diff --git a/.gitattributes b/.gitattributes index 62abbfcddcae2e28cb4b119653c1709e5d474a9f..500964c76ef5f54e794f65d1bc64ac441a47d103 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1076,3 +1076,186 @@ FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_ FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_500_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/files/media/videos/val0/reconstructed_500_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_DF100_SINK3_WINDOW9/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink3_window9_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_df100/wandb/run-20260425_142602-3fi9xe8g/run-3fi9xe8g.wandb filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000100.fake_score_model/.metadata filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000100.fake_score_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000100.fake_score_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000100.fake_score_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000100.fake_score_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000100.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000100.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000100.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000100.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000100.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000200.fake_score_model/.metadata filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000200.fake_score_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000200.fake_score_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000200.fake_score_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000200.fake_score_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000200.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000200.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000200.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000200.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000200.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000300.fake_score_model/.metadata filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000300.fake_score_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000300.fake_score_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000300.fake_score_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000300.fake_score_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000300.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000300.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000300.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000300.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000300.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000400.fake_score_model/.metadata filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000400.fake_score_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000400.fake_score_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000400.fake_score_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000400.fake_score_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000400.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000400.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000400.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000400.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000400.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_model/.metadata filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_optim/.metadata filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_optim/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_optim/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_optim/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_optim/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_optim/.metadata filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_optim/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_optim/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_optim/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_optim/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.fake_score_model/.metadata filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.fake_score_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.fake_score_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.fake_score_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.fake_score_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/train_media/data/real_100_47403c800591fce4a99b.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/train_media/data/real_200_4e05a85e239f00a00e38.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/train_media/data/real_300_97779d11e00d4fea84a0.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/train_media/data/real_400_112c0905448a821974f0.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/train_media/data/real_500_2594e32c120a620ede9d.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/train_media/data/real_600_aebf0f3e2314a893fcf4.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/train_media/student/generation_100_997404f2a352c169efb9.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/train_media/student/generation_200_a8d20e0c527978b248bb.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/train_media/student/generation_300_60813195825a9a04e33f.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/train_media/student/generation_400_a68193167328f5696a93.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/train_media/student/generation_500_1980c9ab0dbf9a6419fd.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/train_media/student/generation_600_77974b3b30df0a6f7c8d.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_100_23af72f047530bfd7798.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_100_2bd0d4bf113f67aa61dc.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_100_58bd95e445782cc6497f.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_100_64f915fab9d30c69fc5d.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_100_70a1c09fb071863c2781.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_100_80066741d1c924702f31.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_100_9171257db4ff8d9ac88f.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_100_a1b3af987cf0466e1416.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_100_a9ece02216b04a6c80ec.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_100_d293e0655da976f44ed1.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_200_35dca6407d00391ae2ec.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_200_56df9f3acc292d64bb66.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_200_664cc59371c2aab5485d.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_200_7d505b1ef380e966a0b7.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_200_9289a0940a968c587b6b.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_200_abddffb1342b044d3385.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_200_ba6034c21748ba0e2e9e.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_200_e079f7563eb4fe359860.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_200_f0e59524ae186790606e.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_200_f3083592b5ff34d16d81.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_300_2c7461b188ae29258ff1.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_300_4a688a18e2baeddce8a2.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_300_6a59a4aa7c0460b697a2.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_300_72a20e7bf6b86592041c.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_300_74625a6884cfa2ec1fef.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_300_76a61b7a843f66955e5d.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_300_7a6a0dc57a7ff4e91e5c.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_300_8eefc192414073c5f820.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_300_a6446a3956e658eec13a.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_300_ec9286082e717a1205b8.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_400_27d7fd095fc078c0cd62.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_400_31086ae2d0a06bc7d879.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_400_3df718c3fe35349ca31f.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_400_6b5c23dd513308fff355.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_400_85d43cd6035ca1df9373.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_400_88b56ab0b4b519af06c6.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_400_9d8083354deeb4484f05.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_400_b603c5057580a7a5535f.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_400_e7d287c535cccd517383.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_400_e8906324008716057975.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_500_268f148fe52acaf11e20.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_500_37b567e82972db0905e6.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_500_51967ebf199a3fc5b970.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_500_5affe373089cb1c23cfd.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_500_6f432ba8a179d2cf9d42.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_500_ad8196e3454985113065.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_500_bc93c864adf844e76807.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_500_ecf33b81c961fa87b44c.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_500_eff1a043cc25e64d5626.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_500_f437f159bf41584702e6.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_100_15ac5b765a401d035db1.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_100_294040ac378ceaa34329.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_100_6ff046418b5d41477989.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_100_80aab00fcfa85d5db98e.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_100_884b2d197a5511393680.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_100_a076d17e202ed18ebdf8.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_100_c0196720a45889bb2198.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_100_c0f892b5fba78348af85.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_100_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_100_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_200_15ac5b765a401d035db1.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_200_294040ac378ceaa34329.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_200_6ff046418b5d41477989.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_200_80aab00fcfa85d5db98e.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_200_884b2d197a5511393680.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_200_a076d17e202ed18ebdf8.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_200_c0196720a45889bb2198.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_200_c0f892b5fba78348af85.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_200_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_200_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_300_15ac5b765a401d035db1.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_300_294040ac378ceaa34329.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_300_6ff046418b5d41477989.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_300_80aab00fcfa85d5db98e.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_300_884b2d197a5511393680.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_300_a076d17e202ed18ebdf8.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_300_c0196720a45889bb2198.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_300_c0f892b5fba78348af85.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_300_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_300_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_400_15ac5b765a401d035db1.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_400_294040ac378ceaa34329.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_400_6ff046418b5d41477989.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_400_80aab00fcfa85d5db98e.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_400_884b2d197a5511393680.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_400_a076d17e202ed18ebdf8.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_400_c0196720a45889bb2198.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_400_c0f892b5fba78348af85.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_400_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_400_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_500_15ac5b765a401d035db1.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_500_294040ac378ceaa34329.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_500_6ff046418b5d41477989.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_500_80aab00fcfa85d5db98e.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_500_884b2d197a5511393680.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_500_a076d17e202ed18ebdf8.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_500_c0196720a45889bb2198.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_500_c0f892b5fba78348af85.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_500_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_500_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text +hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/run-lvwg9pfc.wandb filter=lfs diff=lfs merge=lfs -text diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000100.fake_score_model/.metadata b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000100.fake_score_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..6d17a87cc61e5a0ba4ece9f4ee99603ec0e92b91 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000100.fake_score_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95fa4dd5e6f37f0ac30508bfbdc44cb2d81134bddbe306201148b3e5d7a33a87 +size 635521 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000100.fake_score_model/__0_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000100.fake_score_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..0cf279a3b300410633093efec7254c3c004ed86c --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000100.fake_score_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bdd89229a0e5f4e8fc3769a47b325fc99224801ecf46ec7300794fcc6d0d4aa8 +size 1423573889 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000100.fake_score_model/__1_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000100.fake_score_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..974282977ab1e6555ff0605e45bd6d9b91001d1f --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000100.fake_score_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:773532e5affd879fc913a0e075b17198ae4e8b1d98d9b2eadbf1c92cca4de92d +size 1422406794 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000100.fake_score_model/__2_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000100.fake_score_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..d7588763f37706ea14086cd4fac55aee06f8b251 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000100.fake_score_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d459012e3cd52d2034c7adbb9e8c0b4696048df17c5033b396dc2de86e27669 +size 1422431002 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000100.fake_score_model/__3_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000100.fake_score_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..9eead10e070f7af4ea4800848cb24b4a2b8796f4 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000100.fake_score_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bdd776ab29115ef08156f94d2a1575513bb226a3658fb293bd9ac298a0bae7c8 +size 1422429489 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000100.net_model/.metadata b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000100.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..757d3b4376bfcaf5477e7d675aeee380ac914da0 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000100.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fa878d9e706492fe53534f18142f7b6bbcc05793e0566e3c0c615b8e4f106e1 +size 635514 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000100.net_model/__0_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000100.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..ef3ebf9d7c96fb0fc01d3d43d1a96be90830095b --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000100.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a797129055948efd79db282c8b49015f194e5c82dae55ad8a169bb60b0c283a +size 1423573889 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000100.net_model/__1_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000100.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..88cfcd840011da8b7796dbe9c1c0a9ac3fe7daf6 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000100.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9202432f2b106bc50ca12b565aafd5631247d259ad34b97883c2883abed5f42 +size 1422406794 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000100.net_model/__2_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000100.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..873665e9ea22bcaba0832dfafefc23afb0e3a460 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000100.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7296dbc5fe6c0d4ed9375364f1b4a4580165826202151fba3c173b6edaf8c5cf +size 1422431002 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000100.net_model/__3_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000100.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..90e8efd0a5f2102c8d062013d2c4f847f72b281c --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000100.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42252f1531088dbd9f574209ee7f00f8d43253419d59817c9e4c962d884a3c58 +size 1422429489 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000100.pth b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000100.pth new file mode 100644 index 0000000000000000000000000000000000000000..0f637e0f6792d8ae1960e0e448573321eacbddc4 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000100.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c9e7bf7b483afea9b5f5fc9f6d1a8368eea2246e5e5404c08b9fba8acdc0064 +size 1901 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000200.fake_score_model/.metadata b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000200.fake_score_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..cb0832ae538d67b9e69a55b014c4a997e5ddc6e9 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000200.fake_score_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e698913a8207c8d9669df5d8808ecca05f34c2e671d65ba2efc41fe9ed655c3b +size 635521 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000200.fake_score_model/__0_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000200.fake_score_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..c9af212875ffe4d63a57364ad8ef76444d18810d --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000200.fake_score_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6119d584f6d32b2e292950eead89624f2314ca5dc4d8930d5f4e7f1f7b6ac9ee +size 1423573889 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000200.fake_score_model/__1_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000200.fake_score_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..61fd72a6888d78e6584da8d06584f85b16282ea7 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000200.fake_score_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:090440106bb72f3439197a33cde0e415eb96257694e73069ea05b0f0a354cbb2 +size 1422406794 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000200.fake_score_model/__2_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000200.fake_score_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..c0576ec1b9f2ce7518308cf280e9f2831820810a --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000200.fake_score_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3926de023bf7d004f91777f2b85b0136833d2e3543e32d26e508e3c89c0c1024 +size 1422431002 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000200.fake_score_model/__3_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000200.fake_score_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..ec29adc5df50d55e950e286df8abc02f443ebb34 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000200.fake_score_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a19aa5ea8861e5a79051ac89aa362b8529f69b5782e0faa3e91678b5a39fe0a +size 1422429489 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000200.net_model/.metadata b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000200.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..4a46e5464e2fc3a7c1f1a8b9805391a8c61480ac --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000200.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1899ad831912628925c19b17a5018e05e866f1dd45bb835b7efe6ccf9020015 +size 635514 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000200.net_model/__0_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000200.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..ba763d60ee2f3cc01ec4765bd0afb45a8d1daf74 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000200.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9390b9d7e0a02713f9b9a4550d698fc9cc17de80609c59dccbda9ad30b1d0380 +size 1423573889 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000200.net_model/__1_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000200.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..710b4b0d4ea6619bd74523c9f25fa33a1c9b4234 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000200.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9fffbf9cbcc7e75008e70ca88b8cbfa04e0e45573218f8c5da2a05c4b0a35bf9 +size 1422406794 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000200.net_model/__2_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000200.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..31f80b31a79770b9ed750d4f18e5f434582b0619 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000200.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:364292ff58a49571e75170e1f210d00987ae28ed8439df98549d97a3fdbb6d03 +size 1422431002 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000200.net_model/__3_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000200.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..6fc90c24c5681bea5abbf4e6c8b3f2550c90cf40 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000200.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c411a2b070b1cac55e75e18df9c33882e8517535cbb04930168eb6057590af93 +size 1422429489 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000200.pth b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000200.pth new file mode 100644 index 0000000000000000000000000000000000000000..94689c6d88777305ec14be3e82dcf676c28c81a1 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000200.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18d49c29dddf4cff9abe84c331dfad17799824bf4158c145f77812c5430228f1 +size 1901 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000300.fake_score_model/.metadata b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000300.fake_score_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..f74b3f9240a2964a2eb1fc5eb53745ea7dce09ab --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000300.fake_score_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3b491c9da338ac1061d2bf726de5dbfef1d3fa68aafe92b81efc230f1d05d5c +size 635521 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000300.fake_score_model/__0_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000300.fake_score_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..1988d2013b200f7e73021872d0c0bf14c63ec40a --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000300.fake_score_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:40d47dcdff708a66b37ff7e813369d53532cc0fc7e022eed4a11e742bd4d0eb0 +size 1423573889 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000300.fake_score_model/__1_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000300.fake_score_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..610b0835d65ec7feb0fdff52a2af802dd63fb2f6 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000300.fake_score_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fba83d6161518e0ff8ee82e77ec85e470ff7e71cad271823f4b10daea4c80ecc +size 1422406794 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000300.fake_score_model/__2_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000300.fake_score_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..dbfd8e9a4bdb5de27d99a71dfb18f4dbe02ae232 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000300.fake_score_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4915cbe15194ac0a68b821fa092cad43191890cb9ca4e68609c9ccdaa3aa5a2f +size 1422431002 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000300.fake_score_model/__3_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000300.fake_score_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..a20e761e323a421c260af21a8f16520356abceb3 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000300.fake_score_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8854513333dac49e9a1b368d6f7ab6e184ce743129520c60f2dd891bd788460 +size 1422429489 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000300.net_model/.metadata b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000300.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..8e09265d3f0998f4acd924a28667f2ed7b76676f --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000300.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b24c2cd6df79411558304b3ce893d44637171a696150e3707d369570f5485b4f +size 635514 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000300.net_model/__0_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000300.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..73e059c21e99eee6620275c28c7843d1d4c3b018 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000300.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09812a6df8985f109eecf11fc017e596478fecbb79562986325dc41bd616844e +size 1423573889 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000300.net_model/__1_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000300.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..1b3fc514b03244e0f029c7e1ec57005599faa1c6 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000300.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f408daeaecb2cfec610ced809b58508b818e9b4cdec24074f2872da287de3033 +size 1422406794 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000300.net_model/__2_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000300.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..bba8aa44d3d6594da8c75813008caba6a9bb119d --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000300.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ad94e4a147d8da4c491857a8f3c395dcd413b4d0e8c474e4f5a847915ae2338 +size 1422431002 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000300.net_model/__3_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000300.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..6d1a58d53b90dd0a604efb177eed10e7f02bdb92 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000300.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4636b40dc3bc909858c7e2e698f0b2b120b79ca39ded98d90fd1bd106feabe65 +size 1422429489 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000300.pth b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000300.pth new file mode 100644 index 0000000000000000000000000000000000000000..36cdfef63708c11e993e5164fd9859eb64d6d33e --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000300.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:463104a57d6bfde142e5cd01a059afdb8b99dad40c73e04a236502ee34fb4d66 +size 1901 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000400.fake_score_model/.metadata b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000400.fake_score_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..f0ac7d7251be62a51533aa07b5e543845f89b4ec --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000400.fake_score_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f909dd4ee2215682d5ee2c481fb49396e3a981926cd8d790d6ebe924f3f85732 +size 635521 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000400.fake_score_model/__0_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000400.fake_score_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..2540080e5483986f2b1d4a7d628dfff536da78af --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000400.fake_score_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d2804c421d2655121ca1b5ca83ab66e8acfe92541468b8f9f0353a622b513889 +size 1423573889 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000400.fake_score_model/__1_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000400.fake_score_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..600dc1ff1a079c25c2433be6e64a84e525d8d539 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000400.fake_score_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4cffd5220223627bee30b1365cb5825c3df343e1d7d1bc907fd19df838feecd0 +size 1422406794 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000400.fake_score_model/__2_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000400.fake_score_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..770a81dbeae2088a26459cb14d17084be42e9235 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000400.fake_score_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3105482d85ddf66b86599a6d71ee3c4a0a9ee0e3de26f0f3fcfc6797a70162f1 +size 1422431002 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000400.fake_score_model/__3_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000400.fake_score_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..205219aa8b73882ec7c15e84358c8b8bfc380b61 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000400.fake_score_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:789d1eb1d121d43b49d1667e99668251a5178be099a4c1331b548dc8fc80cf1f +size 1422429489 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000400.net_model/.metadata b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000400.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..72cecd70599ad6835f4db80aad10ab43edc286e9 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000400.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c7289bcf6dd5367a4841ec3ba5d068d5aa492627e2d435f5f47a5e6df38c667 +size 635514 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000400.net_model/__0_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000400.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..a15d10cfe45badc494b509fada7ad7384fd6a27a --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000400.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d91938eaa8bd15164885a85c724236c69d7c2a898202461143b091a0277ecb3 +size 1423573889 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000400.net_model/__1_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000400.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..0b185d323690b9b989ac4468913145e4f3d05c29 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000400.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f85da4b062b005f87e970413b23da03b4e20e446df736d9b710bbcef5cc03969 +size 1422406794 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000400.net_model/__2_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000400.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..40bc1448c76c19c079f679a15135f64f1294fdc6 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000400.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c3fa2215e2f3ec6e93fe523cddc18abf6f9d5a02c39e7a6fc7115231735812b6 +size 1422431002 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000400.net_model/__3_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000400.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..53652c1b3ae3d60b1576d0f13f1a3266ac5bdf69 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000400.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0602c66cab66f61046f03099578940f6f00bbf1f04028980c69546a3bcda50a9 +size 1422429489 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000400.pth b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000400.pth new file mode 100644 index 0000000000000000000000000000000000000000..0deb1499fd51197d9815f644a93381fbe0678161 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000400.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:31545bbbd2979dddf082789dbf999d9324dc9febf1b016304ddc713ede429c3e +size 1901 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_model/.metadata b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..6c34edd3d8585e934c0391038e7d1339d6896810 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ebd96da76380d09ad7da826875ca48a50fe52be87cfc521c6f806999c574278 +size 635521 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_model/__0_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..596337083608c1641b1bf613da6dab59218ea55c --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58bde1b7a53ef209bef6825139c269cda29e6d1ef57bf49453bdbe736759be88 +size 1423573889 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_model/__1_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..9107db18d1050b90d4d9ed20a516a88659f4f759 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae44916aab01930b59286b5c8cc6c8f8da0d2225c7d67e355759cb0f8becd338 +size 1422406794 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_model/__2_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..9227e656bff4260557d0546031c8f1ea0ddacb69 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6aeafeb5ab8bbc444596671ddcac5787e9b0e8678261fe5ffa6612334a873a62 +size 1422431002 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_model/__3_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..101ba9e705d0c0d27ffb2021e4ee1aba7d6296f4 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eda90de6a86b7ac1470c4caa4e49d5d160c965ced0849d4cb5bc604a0f850035 +size 1422429489 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_optim/.metadata b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_optim/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..d54912f66f7fe2080f24ce6b5c2929bd9566cfac --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_optim/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d718d6c1975b9c6c6a0a28daafae8f2c86ba193d3df613421b24b5de44220bfe +size 1722885 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_optim/__0_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_optim/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..f74001a846b8663b4056d5753e337ad337b215b5 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_optim/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:799df2ec0deba809fca72c2de7730b82dd70a3f01f5b9e7b8f9909d5034936ff +size 2847149355 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_optim/__1_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_optim/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..0e62bcecae9304a42ef3bd8a84f51e0762fe563e --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_optim/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f06c45e24a0a3c4b3d44070105d49b6f8a721e7c78eadf288eb09410e5345e89 +size 2845267132 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_optim/__2_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_optim/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..c01a5460f158c28cd7691e7ffe73c9ce4bb996f7 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_optim/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ae45ff327aa77caa0076cc16ffc8574a744cd77b21b76a2d46acdf502c5fbc0 +size 2845352958 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_optim/__3_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_optim/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..be058517ca075641bb5d45cf781c94a0ffd2147b --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.fake_score_optim/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98702e016fb8bb7bec4884d781adef4ead88908a981bcf19495813b1cbb7cbb3 +size 2845316241 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_model/.metadata b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..1ccf324574d4e91c5f93fdfe05bf8947e55e8b5a --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c49b1425e0aeb56ba2570d67fc8678deea682c1cfb3f2e24804c2a67b52491d9 +size 635514 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_model/__0_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..fbba0989e1a5d6c999630f18dc00a6b7a765a1ad --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61163e4d37bf88c67637a66f0f80ba92e8d0bcb1457dd9f6f17d0c0185c4352f +size 1423573889 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_model/__1_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..f74a1e236525c9574869404a305a1a5052fdce48 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66d41b437fc5b0846c7946a225fee4ecbef89c5efbfd7cdc246eeb86922b8855 +size 1422406794 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_model/__2_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..711c8fece74ae0cbba064b91bb207302fb324031 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00f12e56b4c91f31071b118629e7d9794cba62824146c66a0c5ef698c472ac9d +size 1422431002 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_model/__3_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..932188c9cab741691ae0b460bc5d058e712ea0ff --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92a53a7aa11be9ec395b30a7faa0dc3fd055a1fdefad49398d913a58e433119a +size 1422429489 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_optim/.metadata b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_optim/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..47a92ebc1f35cc18178dfb60c8080a2c32a073b5 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_optim/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad263df2c9b565eb2de53cac51f3ffc6b41408a2f36324063dfb64ed276d2716 +size 1722728 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_optim/__0_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_optim/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..b0db31ffd9b30f74424b5701b517d8cdc22a4ecf --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_optim/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99372bd0c7d65c0f00d9c488bccdd2721b882888f3ad2fcf9ad805a36337f435 +size 2847149355 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_optim/__1_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_optim/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..c10c23cda455872c79f782a88d94eb0d4a59a99f --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_optim/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66f1b380481128a86ced0412e1074f3426d4f278b33f0d9d077114add6b4e469 +size 2845267132 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_optim/__2_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_optim/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..2a22d9ac9505dc64871df7f51f5158af6b08262d --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_optim/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e600340f96e945629f454c661f4dc57c90c05a1df0d9bb823fc7294a68735155 +size 2845352958 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_optim/__3_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_optim/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..36d3a84b3160f5a108fa704c002053134253d921 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.net_optim/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:804c05d5be1ceac9180c6d9dd9c14f04452cc7c7283d98f04f553afd1c087161 +size 2845316241 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.pth b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.pth new file mode 100644 index 0000000000000000000000000000000000000000..df0fb483027e40ac33d6a36f4b38fcfd244d6a44 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a2110994f1f90ec15f3f831251e70d29fe1b83f8a35bb969069c4ac3f211c6d +size 1901 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.fake_score_model/.metadata b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.fake_score_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..f5bbec732ad1c5b60e12ff9675d153724324cbf7 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.fake_score_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b6fcb83d64c83abde1b6e954172cfce9711c03d358235c382b44d1972fec549 +size 635521 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.fake_score_model/__0_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.fake_score_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..577c21d1781cea28c5b74e2a61de3d636ece04f3 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.fake_score_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d4a589a3750b4aed400f53905236d15970866e58bf8bfb25c779bf95251c017 +size 1423573889 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.fake_score_model/__1_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.fake_score_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..d5cd45867ff101d2b4dab83cae08fdf225910332 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.fake_score_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b406a083c0c46fed0380f1c9b2a024c1615b79da10694245e50242530cd204a0 +size 1422406794 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.fake_score_model/__2_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.fake_score_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..6aa20166e8057b6c13f320380e5c8a4d36d2df5f --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.fake_score_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa523d14c10571b9395e1bca00b7b9187043bd4f7ff7923b49c365b9e5f15a37 +size 1422431002 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.fake_score_model/__3_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.fake_score_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..e08f5dff2f672d077850222f34e2409831ce295b --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.fake_score_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e27e2536c710cab2297e5f5eb57441b3d6e1b73f00b80ac8c956d7796fe3c2e +size 1422429489 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.net_model/.metadata b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..4499462cf6f1639c4dd9b4bd224e69b02558af46 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30952216bb22bdf01f4d742ac837ea12f8fac491c637c5776bdc0e7722fa3445 +size 635514 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.net_model/__0_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..01835494d5c05183025b4ce8ac8785c7bf0e54a2 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4032059d9ad1fe9598158dba656847804ed0b092f45b5f9830c7f82ef7530a0e +size 1423573889 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.net_model/__1_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..8b1e3e9f7d7a25ab1c14896437d1417d33a844af --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29923bd36b13729e1e94629f8048493f2901748605148f0ade3c325cca33b249 +size 1422406794 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.net_model/__2_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..05497b506c8eb1d6567ab8fc176b927b8704ddad --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a0f668274d161dad60041ccd6af1af9997682cc9bb3d658a43a8b52294502a3 +size 1422431002 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.net_model/__3_0.distcp b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..84a0e8720f4e1ac0aa5e6475ae78489be7067462 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59561b6e0f34bf749703ba8d843df36e6664e6c2a535b581ee061c80c915d11b +size 1422429489 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.pth b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.pth new file mode 100644 index 0000000000000000000000000000000000000000..a56d721aaffd7a3c9d01c2f790dace95d05fa600 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca3d163bab055381827226140568f3bef7eaac187cebd76878e0b63e9e442356 +size 3 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/config.yaml b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9781c02a6dec73920aa1e500da135dff8da3dd0a --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/config.yaml @@ -0,0 +1,283 @@ +dataloader_train: + _target_: + batch_size: '8' + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_path.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_raw_audio: 'True' + neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt + num_workers: '2' + raw_audio_fps: '25.0' + raw_audio_num_frames: '81' + raw_audio_sample_rate: '16000' + use_ref_sequence: 'True' +dataloader_val: + _target_: + batch_size: '1' + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_val10.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_ode_path: 'False' + neg_text_emb_path: /home/work/stableavatar_data/neg_text_emb.pt + num_workers: '2' + use_ref_sequence: 'True' +eval: + max_ckpt: '100000000' + min_ckpt: '0' + num_samples: '50000' + samples_dir: samples + save_images: 'False' +log_config: + group: omniavatar_sf_full_ft + name: sf_full_ft_t769 + project: OmniAvatar-FastGen + wandb_credential: ./credentials/wandb_api.txt + wandb_entity: paulhcho + wandb_mode: online +model: + add_teacher_to_fsdp_dict: 'True' + center_reward: 'False' + clamp_reward: null + context_noise: '0.0' + ddp_find_unused_parameters: 'True' + device: cuda + discriminator: + _target_: + all_res: + - '32' + - '16' + - '8' + feature_indices: '{0, 1, 2}' + in_channels: '256' + discriminator_optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'False' + lr: '0.0001' + model: null + optim_type: adamw + weight_decay: '0.01' + discriminator_scheduler: + _target_: + cycle_lengths: + - '10000000000' + f_max: + - '1.0' + f_min: + - '1.0' + f_start: + - 1e-06 + warm_up_steps: + - '0' + enable_gradient_in_rollout: 'True' + enable_preprocessors: 'True' + fake_score: null + fake_score_net: + _target_: + audio_hidden_size: '32' + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors + in_dim: '65' + merge_lora: 'True' + mode: v2v + model_size: 1.3B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt + schedule_type: rf + use_audio: 'True' + fake_score_optimizer: + _target_: + betas: + - '0.0' + - '0.999' + eps: 1e-08 + fused: 'False' + lr: 2e-06 + model: null + optim_type: adamw + weight_decay: '0.01' + fake_score_pred_type: x0 + fake_score_scheduler: + _target_: + cycle_lengths: + - '10000000000' + f_max: + - '1.0' + f_min: + - '1.0' + f_start: + - 1e-06 + warm_up_steps: + - '0' + fsdp_meta_init: 'False' + gan_loss_weight_gen: '0' + gan_r1_reg_alpha: '0.1' + gan_r1_reg_weight: '0.0' + gan_use_same_t_noise: 'False' + grad_scaler_enabled: 'False' + grad_scaler_growth_interval: '2000' + grad_scaler_init_scale: '65536.0' + guidance_scale: '4.5' + input_shape: + - '16' + - '21' + - '64' + - '64' + last_step_only: 'False' + load_student_weights: 'False' + net: + _target_: + audio_hidden_size: '32' + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors + chunk_size: '3' + in_dim: '65' + local_attn_size: '7' + mode: v2v + model_size: 1.3B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt + schedule_type: rf + sink_size: '1' + total_num_frames: '21' + use_audio: 'True' + use_dynamic_rope: 'True' + net_optimizer: + _target_: + betas: + - '0.0' + - '0.999' + eps: 1e-08 + fused: 'False' + lr: 2e-06 + model: null + optim_type: adamw + weight_decay: '0.01' + net_scheduler: + _target_: + cycle_lengths: + - '10000000000' + f_max: + - '1.0' + f_min: + - '1.0' + f_start: + - 1e-06 + warm_up_steps: + - '0' + precision: bfloat16 + precision_amp: null + precision_amp_enc: null + precision_amp_infer: null + precision_fsdp: float32 + pretrained_model_path: '' + pretrained_student_net_path: '' + reward: + audio_sample_rate: '16000' + checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model + decoder_kind: taew + enabled: 'True' + input_fps: '25.0' + taew_checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth + vshift: '15' + reward_beta: '2' + reward_debug_dir: logs/redmd_debug + reward_weighting_mode: per_sample + same_step_across_blocks: 'True' + sample_t_cfg: + log_t_df: '0.01' + max_t: '0.999' + min_t: '0.001' + shift: '5.0' + t_list: + - '0.999' + - '0.769' + - '0.0' + time_dist_type: shifted + train_p_mean: '-1.1' + train_p_std: '2.0' + save_reward_debug_video: 'False' + skip_layers: null + start_gradient_frame: '0' + student_sample_steps: '2' + student_sample_type: sde + student_update_freq: '5' + teacher: + _target_: + audio_hidden_size: '32' + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors + in_dim: '65' + merge_lora: 'True' + mode: v2v + model_size: 14B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt + schedule_type: rf + use_audio: 'True' + timestep_cfg: + enabled: 'True' + t_hi: '0.882' + t_lo: '0.556' + use_ema: 'False' +model_class: + _target_: + config: null +trainer: + augment_pipe: null + batch_size_global: null + callbacks: + ema: + _target_: + beta: '0.9999' + ema_halflife_kimg: '500' + ema_rampup_ratio: '0.05' + gamma: '16.97' + start_iter: '0' + type: constant + gpu_stats: + _target_: + every_n: '100' + grad_clip: + _target_: + grad_norm: '10.0' + model_key: net + param_count: + _target_: + train_profiler: + _target_: + every_n: '100' + wandb: + _target_: + fps: '25' + sample_logging_iter: '100' + syncnet_checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model + checkpointer: + pretrained_ckpt_key_map: + net: net + pretrained_ckpt_path: /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_t769_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth + s3_container: s3://checkpoints/fastgen + s3_credential: ./credentials/s3.json + save_dir: /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints + use_s3: 'False' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: 'False' + fsdp: 'True' + fsdp_cpu_offload: 'False' + fsdp_min_num_params: '10000000' + fsdp_sharding_group_size: null + global_vars: null + global_vars_val: + - null + grad_accum_rounds: '2' + logging_iter: '1' + max_iter: '5000' + offload_module_in_decoding: 'False' + resume: 'False' + save_ckpt_iter: '100' + seed: '0' + skip_initial_validation: 'True' + tf32_enabled: 'True' + val_seed: null + validation_iter: '100' + visualize_teacher: 'False' diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/debug-internal.log b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..3e213ec5600b26d941826b41f0af25352b2c4da1 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/debug-internal.log @@ -0,0 +1,15 @@ +{"time":"2026-04-30T22:12:05.062483501+09:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-04-30T22:12:05.463604741+09:00","level":"INFO","msg":"stream: created new stream","id":"lvwg9pfc"} +{"time":"2026-04-30T22:12:05.4636515+09:00","level":"INFO","msg":"handler: started","stream_id":"lvwg9pfc"} +{"time":"2026-04-30T22:12:05.464937177+09:00","level":"INFO","msg":"stream: started","id":"lvwg9pfc"} +{"time":"2026-04-30T22:12:05.464958887+09:00","level":"INFO","msg":"sender: started","stream_id":"lvwg9pfc"} +{"time":"2026-04-30T22:12:05.464963248+09:00","level":"INFO","msg":"writer: started","stream_id":"lvwg9pfc"} +{"time":"2026-05-01T10:11:37.611489893+09:00","level":"INFO","msg":"stream: closing","id":"lvwg9pfc"} +{"time":"2026-05-01T10:11:37.617805919+09:00","level":"ERROR","msg":"failed to write to chunk: failed to close: close /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/output.log: no space left on device"} +{"time":"2026-05-01T10:11:37.625534494+09:00","level":"ERROR","msg":"sender: failed to upload run summary: close /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/wandb-summary.json: no space left on device"} +{"time":"2026-05-01T10:11:37.635646188+09:00","level":"ERROR","msg":"sender: failed to upload run config: close /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/config.yaml: no space left on device"} +{"time":"2026-05-01T10:11:39.206389931+09:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-05-01T10:11:39.631573664+09:00","level":"INFO","msg":"handler: closed","stream_id":"lvwg9pfc"} +{"time":"2026-05-01T10:11:39.632071405+09:00","level":"ERROR","msg":"writer: failed closing store: transactionlog: error closing file: close /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/run-lvwg9pfc.wandb: no space left on device"} +{"time":"2026-05-01T10:11:39.633049656+09:00","level":"INFO","msg":"sender: closed","stream_id":"lvwg9pfc"} +{"time":"2026-05-01T10:11:39.633060653+09:00","level":"INFO","msg":"stream: closed","id":"lvwg9pfc"} diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/debug.log b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..cdf1d978e77fdfa59e5c5b6bc2108ec5785bfe8e --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/debug.log @@ -0,0 +1,21 @@ +2026-04-30 22:12:04,810 INFO MainThread:1288706 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-04-30 22:12:04,810 INFO MainThread:1288706 [wandb_setup.py:_flush():81] Configure stats pid to 1288706 +2026-04-30 22:12:04,810 INFO MainThread:1288706 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-04-30 22:12:04,810 INFO MainThread:1288706 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/logs/debug.log +2026-04-30 22:12:04,810 INFO MainThread:1288706 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/logs/debug-internal.log +2026-04-30 22:12:04,810 INFO MainThread:1288706 [wandb_init.py:init():844] calling init triggers +2026-04-30 22:12:04,810 INFO MainThread:1288706 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'log_config': {'project': 'OmniAvatar-FastGen', 'group': 'omniavatar_sf_full_ft', 'name': 'sf_full_ft_t769', 'wandb_mode': 'online', 'wandb_entity': 'paulhcho', 'wandb_credential': './credentials/wandb_api.txt'}, 'trainer': {'cudnn': {'deterministic': 'False', 'benchmark': 'True'}, 'checkpointer': {'save_dir': '/home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints', 'use_s3': 'False', 's3_container': 's3://checkpoints/fastgen', 's3_credential': './credentials/s3.json', 'pretrained_ckpt_path': '/home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_t769_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth', 'pretrained_ckpt_key_map': {'net': 'net'}}, 'callbacks': {'grad_clip': {'grad_norm': '10.0', 'model_key': 'net', '_target_': ""}, 'gpu_stats': {'every_n': '100', '_target_': ""}, 'train_profiler': {'every_n': '100', '_target_': ""}, 'param_count': {'_target_': ""}, 'ema': {'type': 'constant', 'beta': '0.9999', 'gamma': '16.97', 'ema_halflife_kimg': '500', 'ema_rampup_ratio': '0.05', 'start_iter': '0', '_target_': ""}, 'wandb': {'sample_logging_iter': '100', '_target_': "", 'fps': '25', 'syncnet_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model'}}, 'save_ckpt_iter': '100', 'validation_iter': '100', 'skip_initial_validation': 'True', 'logging_iter': '1', 'max_iter': '5000', 'visualize_teacher': 'False', 'seed': '0', 'val_seed': None, 'resume': 'False', 'ddp': 'False', 'fsdp': 'True', 'tf32_enabled': 'True', 'grad_accum_rounds': '2', 'batch_size_global': None, 'offload_module_in_decoding': 'False', 'fsdp_cpu_offload': 'False', 'fsdp_min_num_params': '10000000', 'fsdp_sharding_group_size': None, 'global_vars': None, 'global_vars_val': [None], 'augment_pipe': None}, 'dataloader_train': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_path.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '8', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', '_target_': "", 'load_raw_audio': 'True', 'raw_audio_sample_rate': '16000', 'raw_audio_num_frames': '81', 'raw_audio_fps': '25.0'}, 'dataloader_val': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_val10.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '1', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', 'load_ode_path': 'False', '_target_': ''}, 'eval': {'num_samples': '50000', 'save_images': 'False', 'min_ckpt': '0', 'max_ckpt': '100000000', 'samples_dir': 'samples'}, 'model': {'net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'chunk_size': '3', 'total_num_frames': '21', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt', 'net_pred_type': 'flow', 'schedule_type': 'rf', 'local_attn_size': '7', 'sink_size': '1', 'use_dynamic_rope': 'True', '_target_': ""}, 'teacher': {'model_size': '14B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'fake_score_net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'guidance_scale': '4.5', 'timestep_cfg': {'enabled': 'True', 't_lo': '0.556', 't_hi': '0.882'}, 'skip_layers': None, 'net_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'net_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'sample_t_cfg': {'time_dist_type': 'shifted', 'train_p_mean': '-1.1', 'train_p_std': '2.0', 'shift': '5.0', 'min_t': '0.001', 'max_t': '0.999', 't_list': ['0.999', '0.769', '0.0'], 'log_t_df': '0.01'}, 'input_shape': ['16', '21', '64', '64'], 'device': 'cuda', 'grad_scaler_enabled': 'False', 'grad_scaler_init_scale': '65536.0', 'grad_scaler_growth_interval': '2000', 'pretrained_model_path': '', 'pretrained_student_net_path': '', 'load_student_weights': 'False', 'enable_preprocessors': 'True', 'use_ema': 'False', 'student_sample_steps': '2', 'student_sample_type': 'sde', 'fsdp_meta_init': 'False', 'add_teacher_to_fsdp_dict': 'True', 'ddp_find_unused_parameters': 'True', 'precision': 'bfloat16', 'precision_amp': None, 'precision_amp_infer': None, 'precision_amp_enc': None, 'precision_fsdp': 'float32', 'fake_score_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'fake_score_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'discriminator': {'feature_indices': '{0, 1, 2}', 'all_res': ['32', '16', '8'], 'in_channels': '256', '_target_': ""}, 'discriminator_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '0.0001', 'weight_decay': '0.01', 'betas': ['0.9', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'discriminator_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'student_update_freq': '5', 'gan_loss_weight_gen': '0', 'gan_use_same_t_noise': 'False', 'fake_score_pred_type': 'x0', 'gan_r1_reg_weight': '0.0', 'gan_r1_reg_alpha': '0.1', 'enable_gradient_in_rollout': 'True', 'start_gradient_frame': '0', 'same_step_across_blocks': 'True', 'last_step_only': 'False', 'context_noise': '0.0', 'fake_score': None, 'reward': {'enabled': 'True', 'checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model', 'input_fps': '25.0', 'audio_sample_rate': '16000', 'vshift': '15', 'decoder_kind': 'taew', 'taew_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth'}, 'reward_beta': '2', 'center_reward': 'False', 'clamp_reward': None, 'reward_weighting_mode': 'per_sample', 'save_reward_debug_video': 'False', 'reward_debug_dir': 'logs/redmd_debug'}, 'model_class': {'config': None, '_target_': ""}, '_wandb': {}} +2026-04-30 22:12:04,810 INFO MainThread:1288706 [wandb_init.py:init():892] starting backend +2026-04-30 22:12:05,046 INFO MainThread:1288706 [wandb_init.py:init():895] sending inform_init request +2026-04-30 22:12:05,059 INFO MainThread:1288706 [wandb_init.py:init():903] backend started and connected +2026-04-30 22:12:05,063 INFO MainThread:1288706 [wandb_init.py:init():973] updated telemetry +2026-04-30 22:12:05,082 INFO MainThread:1288706 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-04-30 22:12:06,169 INFO MainThread:1288706 [wandb_init.py:init():1042] starting run threads in backend +2026-04-30 22:12:06,416 INFO MainThread:1288706 [wandb_run.py:_console_start():2524] atexit reg +2026-04-30 22:12:06,416 INFO MainThread:1288706 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-04-30 22:12:06,416 INFO MainThread:1288706 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-04-30 22:12:06,416 INFO MainThread:1288706 [wandb_run.py:_redirect():2465] Redirects installed. +2026-04-30 22:12:06,420 INFO MainThread:1288706 [wandb_init.py:init():1082] run started, returning control to user process +2026-05-01 10:11:37,611 INFO wandb-AsyncioManager-main:1288706 [service_client.py:_forward_responses():134] Reached EOF. +2026-05-01 10:11:37,611 INFO wandb-AsyncioManager-main:1288706 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles. diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/config.yaml b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/train_media/data/real_100_47403c800591fce4a99b.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/train_media/data/real_100_47403c800591fce4a99b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23f2f377ccf704bcaa09a4c9b1a9b5151ab73aac --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/train_media/data/real_100_47403c800591fce4a99b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47403c800591fce4a99b38622b7607baeec11f4e025df3962d292acf880fba6c +size 192248 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/train_media/data/real_200_4e05a85e239f00a00e38.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/train_media/data/real_200_4e05a85e239f00a00e38.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c0fdfbad50d1423fb0ee98a75767d17d3c13b75b --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/train_media/data/real_200_4e05a85e239f00a00e38.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e05a85e239f00a00e386c8d8a9eaabeac26dd4943226f41d80d9f6b7c50bff6 +size 181830 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/train_media/data/real_300_97779d11e00d4fea84a0.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/train_media/data/real_300_97779d11e00d4fea84a0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1fd7a958d162e551e0f6ce9b94e075ce8f98807e --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/train_media/data/real_300_97779d11e00d4fea84a0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97779d11e00d4fea84a00b1667ca4d081ba87b85cf65fac40d228e9774f762d2 +size 194172 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/train_media/data/real_400_112c0905448a821974f0.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/train_media/data/real_400_112c0905448a821974f0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9684c7bde9ed18086d39e7a9880b642c505b3947 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/train_media/data/real_400_112c0905448a821974f0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:112c0905448a821974f0bf0a94d9107473bb02b60e0ebe1e75a57d309670facb +size 219311 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/train_media/data/real_500_2594e32c120a620ede9d.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/train_media/data/real_500_2594e32c120a620ede9d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7e38036c10e966c2c82fa717b29431a1ff88e1ae --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/train_media/data/real_500_2594e32c120a620ede9d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2594e32c120a620ede9d4613c87c8985d5ec03a9250077d94093cf8001922aae +size 190461 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/train_media/data/real_600_aebf0f3e2314a893fcf4.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/train_media/data/real_600_aebf0f3e2314a893fcf4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..df584863079059c25acb730ed404066fdef5d340 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/train_media/data/real_600_aebf0f3e2314a893fcf4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aebf0f3e2314a893fcf47216a0d6c68b02b30d80a7347c759681d9497a805c01 +size 220015 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/train_media/student/generation_100_997404f2a352c169efb9.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/train_media/student/generation_100_997404f2a352c169efb9.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fc243bd78e22f6c25a9b9e81bf5a1c6df3c2f575 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/train_media/student/generation_100_997404f2a352c169efb9.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:997404f2a352c169efb9f09ee3c6c4a14c4a0f7c6f50fc92cc684904066cf185 +size 189380 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/train_media/student/generation_200_a8d20e0c527978b248bb.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/train_media/student/generation_200_a8d20e0c527978b248bb.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6a40f1bf106aa4140e41fa4554fbc058f2522346 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/train_media/student/generation_200_a8d20e0c527978b248bb.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8d20e0c527978b248bb48f8eae38873579cb5c0c5a8f8d28578fbdfb720ddd4 +size 190966 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/train_media/student/generation_300_60813195825a9a04e33f.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/train_media/student/generation_300_60813195825a9a04e33f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..caf44a1c65b9c0d781333a7c0f1ebdcebe49d7e7 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/train_media/student/generation_300_60813195825a9a04e33f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60813195825a9a04e33f0f2b177cbf922a3cc299729f23c455cdd420b5a6913e +size 214841 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/train_media/student/generation_400_a68193167328f5696a93.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/train_media/student/generation_400_a68193167328f5696a93.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1a23753cb3623d316c49cd511058f2b7fd017d2c --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/train_media/student/generation_400_a68193167328f5696a93.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a68193167328f5696a93fc668ea049357f7690a92cd6024d5f55527181b81781 +size 236919 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/train_media/student/generation_500_1980c9ab0dbf9a6419fd.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/train_media/student/generation_500_1980c9ab0dbf9a6419fd.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3628ca1a603f0b79901cfdbd7df5e9de334919db --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/train_media/student/generation_500_1980c9ab0dbf9a6419fd.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1980c9ab0dbf9a6419fd43483fc6190916e7ba8040bbc41b2023958e2e5b97cd +size 213724 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/train_media/student/generation_600_77974b3b30df0a6f7c8d.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/train_media/student/generation_600_77974b3b30df0a6f7c8d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..958b6efef46cbb0d77ec640fdf00869ac71bf8d3 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/train_media/student/generation_600_77974b3b30df0a6f7c8d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77974b3b30df0a6f7c8d5a9372ae2a0707f086d0d97ca6d65aec402a77dbea3c +size 252862 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_100_23af72f047530bfd7798.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_100_23af72f047530bfd7798.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fe1d6207e1cebf32b879fa048895a944a5a14c0e --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_100_23af72f047530bfd7798.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23af72f047530bfd779864cfd141bb629118ff2891d4cc7ff7cca0ca8128a2ec +size 191137 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_100_2bd0d4bf113f67aa61dc.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_100_2bd0d4bf113f67aa61dc.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..dd9d02bbc83bde9250f16666753cceebb53a8364 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_100_2bd0d4bf113f67aa61dc.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2bd0d4bf113f67aa61dcdc6bbd74cf9d5c26179fc6252b824fa4cfde581176cc +size 213892 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_100_58bd95e445782cc6497f.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_100_58bd95e445782cc6497f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7874a85b3d8fd15a0dc35c635d497f309406da8b --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_100_58bd95e445782cc6497f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58bd95e445782cc6497f43d0870d5fc6deef75da5be13ef8576a244f2e483f32 +size 263601 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_100_64f915fab9d30c69fc5d.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_100_64f915fab9d30c69fc5d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..60126e7ee9876e1e7ddb00fa5cc8381b61f3226c --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_100_64f915fab9d30c69fc5d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:64f915fab9d30c69fc5de94b6ce8dc08242ee55d40a4f7e0a7b7f89750a90598 +size 176533 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_100_70a1c09fb071863c2781.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_100_70a1c09fb071863c2781.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b574505a26553262e1de4c1b92fb5deafbf8e7d2 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_100_70a1c09fb071863c2781.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:70a1c09fb071863c2781e114f1f334dd69dd56d0838ded875d47e304a55754b2 +size 216095 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_100_80066741d1c924702f31.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_100_80066741d1c924702f31.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8480940682c5e09229f3b663a3557de4105a6c7e --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_100_80066741d1c924702f31.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80066741d1c924702f31594565f1d4ec32584ac8f665755144f38956715e683e +size 173158 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_100_9171257db4ff8d9ac88f.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_100_9171257db4ff8d9ac88f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..66f428b0efea80a35cdb943a3fe67cface727e20 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_100_9171257db4ff8d9ac88f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9171257db4ff8d9ac88f8af31d236ad12a98ef6eeedd2fd84832c163dfd3a441 +size 220205 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_100_a1b3af987cf0466e1416.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_100_a1b3af987cf0466e1416.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0a64eaa77528ca2734f927c329519d03f8dbecd6 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_100_a1b3af987cf0466e1416.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1b3af987cf0466e14169de0965fd069274931a9b6a8777f63b7536a207aec5f +size 161239 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_100_a9ece02216b04a6c80ec.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_100_a9ece02216b04a6c80ec.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e00692eb1d2916b6a582524931395bb3df10caa5 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_100_a9ece02216b04a6c80ec.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9ece02216b04a6c80ecf7a777c05179f913cffdee633430462e4600b8c24a79 +size 249883 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_100_d293e0655da976f44ed1.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_100_d293e0655da976f44ed1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5de9b32c2e80450dd809257051a3c6c7b9f14d79 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_100_d293e0655da976f44ed1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d293e0655da976f44ed1645c89d4b4402d62a14d16f670dbe06eda43b3cfea6d +size 182355 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_200_35dca6407d00391ae2ec.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_200_35dca6407d00391ae2ec.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5c4a5f7f02c003b554365143b42f826e2ba7b425 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_200_35dca6407d00391ae2ec.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35dca6407d00391ae2ecfde52902cd3bf79bd78d2d50b7f2053367402fe6a5dd +size 165921 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_200_56df9f3acc292d64bb66.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_200_56df9f3acc292d64bb66.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cbfb2871b8b8bba2048a67b6084c0a2c4cf43203 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_200_56df9f3acc292d64bb66.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56df9f3acc292d64bb6696f2f6c8f6cab5b86cdad51065b96b3d5362197cc427 +size 255326 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_200_664cc59371c2aab5485d.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_200_664cc59371c2aab5485d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0f3fb1841f18201025064efe69a90e48dd4350f6 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_200_664cc59371c2aab5485d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:664cc59371c2aab5485d0b992a9707f5942a0c418af7493059ebc72a672f422b +size 219085 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_200_7d505b1ef380e966a0b7.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_200_7d505b1ef380e966a0b7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0e6121dbe1658a54cc5264ea15f311207744d629 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_200_7d505b1ef380e966a0b7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d505b1ef380e966a0b76cb5724c3e5086a585dbacbb1963a60e69675276f13e +size 225700 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_200_9289a0940a968c587b6b.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_200_9289a0940a968c587b6b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1cb523f3dc410e04a112b945cbaf66897d08c759 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_200_9289a0940a968c587b6b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9289a0940a968c587b6b6ffd8b1c852cd77c462750cc53c9c4980b8590882499 +size 266591 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_200_abddffb1342b044d3385.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_200_abddffb1342b044d3385.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7d3983360f9e6fa3d6b9f4d9881a24f1173d690b --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_200_abddffb1342b044d3385.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:abddffb1342b044d3385d45d8305a091bbd80b7b76a3bcb1c0f5b2ee7e9bcbde +size 194882 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_200_ba6034c21748ba0e2e9e.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_200_ba6034c21748ba0e2e9e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..606004cad78046f53af3f280bf15f523be2b6fd4 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_200_ba6034c21748ba0e2e9e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba6034c21748ba0e2e9e24be23c36bea65c17b08642c36a6d9034a83370e2e99 +size 222174 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_200_e079f7563eb4fe359860.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_200_e079f7563eb4fe359860.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2297923101fee69b49db5f335081dcba804b813d --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_200_e079f7563eb4fe359860.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e079f7563eb4fe3598600903372e62d9f697c2237d3f7aa3e20c0df6753f9bca +size 188334 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_200_f0e59524ae186790606e.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_200_f0e59524ae186790606e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c406a0d778cb7f992cc199b63533c9eda26e85be --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_200_f0e59524ae186790606e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0e59524ae186790606e46d78ebc9572296f1d3a5c98d7817f5d878457ba48ca +size 176626 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_200_f3083592b5ff34d16d81.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_200_f3083592b5ff34d16d81.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..346e1f7aeed58915580f929ece673f9eb6aea000 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_200_f3083592b5ff34d16d81.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3083592b5ff34d16d819e1b7fd6f42bb9e38fadfb9e12e1ef0e3762f4c97b6b +size 180897 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_300_2c7461b188ae29258ff1.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_300_2c7461b188ae29258ff1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..52c1f670abda10083d0fe60bdbae932d94a71aff --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_300_2c7461b188ae29258ff1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c7461b188ae29258ff1acb15b9c308f7d014f36ec9a2fb459f74a73e13ab701 +size 285759 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_300_4a688a18e2baeddce8a2.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_300_4a688a18e2baeddce8a2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9c2eb8d124d65bba3a4c1ac18058d2e67c20c9a1 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_300_4a688a18e2baeddce8a2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a688a18e2baeddce8a27a571a2da9eb488d3d38c6ae0cf1305c45622268e57e +size 207471 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_300_6a59a4aa7c0460b697a2.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_300_6a59a4aa7c0460b697a2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..407d7dc9ca77bb3eab6baca352c7d0a694af87e5 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_300_6a59a4aa7c0460b697a2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a59a4aa7c0460b697a2f4e46abf87afbd056c35828ccbdf416c074844a16724 +size 175818 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_300_72a20e7bf6b86592041c.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_300_72a20e7bf6b86592041c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a4714254e39bee40f4005e5d03dec9f4c83c8616 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_300_72a20e7bf6b86592041c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72a20e7bf6b86592041c69c93df3912e5ef26643903538ecdf2b6bdeb122cb0b +size 189993 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_300_74625a6884cfa2ec1fef.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_300_74625a6884cfa2ec1fef.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1b0e76b294763a65454dee7c78d6b78c00ae0afb --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_300_74625a6884cfa2ec1fef.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74625a6884cfa2ec1fefb52f6b68505b165707bece9432953f9e185cbbfb2644 +size 195481 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_300_76a61b7a843f66955e5d.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_300_76a61b7a843f66955e5d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ad5de0c7478018d21f7256165b6621b1494178e9 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_300_76a61b7a843f66955e5d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76a61b7a843f66955e5dfa75d6ac609c004bf5d6e9eaa38b83e7a6d0dd8af74c +size 274935 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_300_7a6a0dc57a7ff4e91e5c.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_300_7a6a0dc57a7ff4e91e5c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cbf8c6da2be38e18851d8fd6fb3c351ae7796cd2 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_300_7a6a0dc57a7ff4e91e5c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a6a0dc57a7ff4e91e5c6b666cde5716f69d175fc08b2e30e1f4a6339926f2f1 +size 228794 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_300_8eefc192414073c5f820.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_300_8eefc192414073c5f820.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..49f937f0adeb8ce98116f5ca0164ed601931e018 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_300_8eefc192414073c5f820.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8eefc192414073c5f820a1fb2fb130d772e2cb483cfbf6aa92a6bc3edecb77ee +size 234938 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_300_a6446a3956e658eec13a.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_300_a6446a3956e658eec13a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..aeb338502720a761860ca74d444c52251763207d --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_300_a6446a3956e658eec13a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a6446a3956e658eec13aae1abe279c8da42cc6963c4aa656d2ef7ed021de5d54 +size 242350 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_300_ec9286082e717a1205b8.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_300_ec9286082e717a1205b8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d069f1e5a203a575d9d1485acf434b521b3209df --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_300_ec9286082e717a1205b8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec9286082e717a1205b81247ed94ad7d1462fa8e7bd07dd99e73f25e77ad91d4 +size 207039 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_400_27d7fd095fc078c0cd62.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_400_27d7fd095fc078c0cd62.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..915032ce0ef9233af99c5c44336d56eae1f4fec0 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_400_27d7fd095fc078c0cd62.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27d7fd095fc078c0cd629ff13767e5d69c68bcaf2600f3b25602fc8726c0b9b5 +size 280604 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_400_31086ae2d0a06bc7d879.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_400_31086ae2d0a06bc7d879.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1ab7920bb118a42b1a77014c2cb8e360b22bc1db --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_400_31086ae2d0a06bc7d879.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:31086ae2d0a06bc7d879b90edc1bad18f31343ea9117a23be00ed4670d1b9e52 +size 281226 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_400_3df718c3fe35349ca31f.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_400_3df718c3fe35349ca31f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3d04602ce8ee6de91ee2c7d05cc044e900f640db --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_400_3df718c3fe35349ca31f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3df718c3fe35349ca31fa8ef9bc3fe5bc917756896453f7ee20222256833ad6e +size 208690 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_400_6b5c23dd513308fff355.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_400_6b5c23dd513308fff355.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9124f4437eef403fc535562be1476407de673281 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_400_6b5c23dd513308fff355.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b5c23dd513308fff355751256c5accc55c5df83269af517605442c2c5f7a058 +size 195250 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_400_85d43cd6035ca1df9373.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_400_85d43cd6035ca1df9373.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e33b6f013904e543ea80173bb828d909e97820df --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_400_85d43cd6035ca1df9373.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85d43cd6035ca1df9373acda2a8cd362278c2b69cac107997db1a5696bfff167 +size 234525 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_400_88b56ab0b4b519af06c6.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_400_88b56ab0b4b519af06c6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..702a20d9c917eb5911c1efc24b3a43a79913bf86 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_400_88b56ab0b4b519af06c6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88b56ab0b4b519af06c6cbce0cb558c5d5f8f8881f2baeb5b976bda6bf4982b0 +size 178035 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_400_9d8083354deeb4484f05.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_400_9d8083354deeb4484f05.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a1ca8c1700a5373a99232257e9c0cf43a73ce8fc --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_400_9d8083354deeb4484f05.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d8083354deeb4484f0509c1b3c025d3cc1191ce6417379681e6c83de301251f +size 204822 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_400_b603c5057580a7a5535f.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_400_b603c5057580a7a5535f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..92479026fff87e292706c37067f93fc6195e1f71 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_400_b603c5057580a7a5535f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b603c5057580a7a5535f076b0281cbc032b71cd9cf54cb05893f79bbbece61ba +size 230218 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_400_e7d287c535cccd517383.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_400_e7d287c535cccd517383.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..91b9f334e3dc64f2f5ee6918da94500e719cbf0e --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_400_e7d287c535cccd517383.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7d287c535cccd51738399e45e89ff6b312a1b01bd35825da39aa60816da8eec +size 191639 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_400_e8906324008716057975.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_400_e8906324008716057975.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..36ce94f6c64cb91b4c73bbcce3aa476daf785c76 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_400_e8906324008716057975.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8906324008716057975e6fdf5119224097ecc68242d5bed43eaa8301da56af4 +size 243599 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_500_268f148fe52acaf11e20.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_500_268f148fe52acaf11e20.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..90623fe7d39c47bc6a4dc9895172213a47e9859e --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_500_268f148fe52acaf11e20.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:268f148fe52acaf11e20aac1a1af12aa500c3136fd51aa19e8a5372e3b335ba0 +size 253354 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_500_37b567e82972db0905e6.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_500_37b567e82972db0905e6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ae2e86b8195a40bbdafc991223b655b8279d8610 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_500_37b567e82972db0905e6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37b567e82972db0905e63ee297bd205aee77169d5955ca8aa7da16cc837e5d82 +size 181831 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_500_51967ebf199a3fc5b970.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_500_51967ebf199a3fc5b970.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3aceadcf3b6c2c46da65038496f3c9f6cb72321b --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_500_51967ebf199a3fc5b970.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51967ebf199a3fc5b970cbe5dac9566b2ae3e302a7290cf6327c1f6d32c8a8c1 +size 195662 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_500_5affe373089cb1c23cfd.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_500_5affe373089cb1c23cfd.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ad576c2e64cf7d35bc8fb6055c76be230224407e --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_500_5affe373089cb1c23cfd.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5affe373089cb1c23cfd97e53a17ce81db46ece977d94e2188a8a312434eda8c +size 236735 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_500_6f432ba8a179d2cf9d42.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_500_6f432ba8a179d2cf9d42.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2873b5239cfbbe06d8c7488983eaf60c0fb6e75e --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_500_6f432ba8a179d2cf9d42.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f432ba8a179d2cf9d424d79bf07b4ade10a28b3c1cfd89e468d7f81dd090bab +size 213943 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_500_ad8196e3454985113065.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_500_ad8196e3454985113065.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6b02916fafa66b2b244d11741cba712f48519af2 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_500_ad8196e3454985113065.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad8196e3454985113065ef2de79c81e01214f8c3fc85ebcabe9fc759536ceed6 +size 294877 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_500_bc93c864adf844e76807.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_500_bc93c864adf844e76807.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2faa281c829cd95ab2f77a6347e2177ae30dfe6d --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_500_bc93c864adf844e76807.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc93c864adf844e7680719f2c5c40b0d77edadcb4d6fb7ca06e59fb3e7d2dad1 +size 199544 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_500_ecf33b81c961fa87b44c.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_500_ecf33b81c961fa87b44c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9a3fc98c207e28b0e7debd71ab5dd92162e98ca1 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_500_ecf33b81c961fa87b44c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ecf33b81c961fa87b44cec99381f95c1cde72a83f587d235c94a50f5aac66a23 +size 209988 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_500_eff1a043cc25e64d5626.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_500_eff1a043cc25e64d5626.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fef37eeb858885b466d390e32f8344c6658b2783 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_500_eff1a043cc25e64d5626.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eff1a043cc25e64d562643c98c268fe4199cdcf622bcdbc1a707ab01e77da6e2 +size 289550 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_500_f437f159bf41584702e6.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_500_f437f159bf41584702e6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..eeea6e4c4f4f1bcc5e736e3ef5c735126a42b31a --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/generated_500_f437f159bf41584702e6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f437f159bf41584702e6d6b26b5dd25d83f4534fa1f24ee9177ac348517df428 +size 240818 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_100_15ac5b765a401d035db1.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_100_15ac5b765a401d035db1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39d693cdae1da9a3a67729fa63439503876eafbc --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_100_15ac5b765a401d035db1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117 +size 190004 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_100_294040ac378ceaa34329.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_100_294040ac378ceaa34329.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23dc14931dfca40fe3f7d6b1fdefe0a1edca1ab9 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_100_294040ac378ceaa34329.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c +size 249372 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_100_6ff046418b5d41477989.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_100_6ff046418b5d41477989.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95ab423dcc64a40477b5d8f2e5d44351e2781109 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_100_6ff046418b5d41477989.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac +size 214001 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_100_80aab00fcfa85d5db98e.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_100_80aab00fcfa85d5db98e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09ef3a36f7d3e07e8b458916212891367b079325 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_100_80aab00fcfa85d5db98e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327 +size 182028 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_100_884b2d197a5511393680.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_100_884b2d197a5511393680.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..550cb7c37290fce2f4554ad8f26113320700bfa3 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_100_884b2d197a5511393680.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214 +size 170922 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_100_a076d17e202ed18ebdf8.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_100_a076d17e202ed18ebdf8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c59919de964bcf66a3cfe53f52bedd0a8b91df2e --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_100_a076d17e202ed18ebdf8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb +size 165516 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_100_c0196720a45889bb2198.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_100_c0196720a45889bb2198.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018bd36397d571746795d1f4619cc8f4fc2766da --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_100_c0196720a45889bb2198.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8 +size 173236 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_100_c0f892b5fba78348af85.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_100_c0f892b5fba78348af85.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f296c5f3aec5b609e307e17bbeffa25264deb91 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_100_c0f892b5fba78348af85.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8 +size 222986 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_100_f5463a4ea6f68a39c5d0.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_100_f5463a4ea6f68a39c5d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..078934eedc27e8de1e5fed0272335f707868e553 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_100_f5463a4ea6f68a39c5d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395 +size 164630 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_100_f7412c9e170ffd299748.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_100_f7412c9e170ffd299748.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eaf7bba0c634cee305854ae2bbd756c90092d38 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_100_f7412c9e170ffd299748.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273 +size 227490 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_200_15ac5b765a401d035db1.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_200_15ac5b765a401d035db1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39d693cdae1da9a3a67729fa63439503876eafbc --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_200_15ac5b765a401d035db1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117 +size 190004 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_200_294040ac378ceaa34329.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_200_294040ac378ceaa34329.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23dc14931dfca40fe3f7d6b1fdefe0a1edca1ab9 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_200_294040ac378ceaa34329.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c +size 249372 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_200_6ff046418b5d41477989.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_200_6ff046418b5d41477989.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95ab423dcc64a40477b5d8f2e5d44351e2781109 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_200_6ff046418b5d41477989.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac +size 214001 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_200_80aab00fcfa85d5db98e.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_200_80aab00fcfa85d5db98e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09ef3a36f7d3e07e8b458916212891367b079325 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_200_80aab00fcfa85d5db98e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327 +size 182028 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_200_884b2d197a5511393680.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_200_884b2d197a5511393680.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..550cb7c37290fce2f4554ad8f26113320700bfa3 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_200_884b2d197a5511393680.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214 +size 170922 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_200_a076d17e202ed18ebdf8.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_200_a076d17e202ed18ebdf8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c59919de964bcf66a3cfe53f52bedd0a8b91df2e --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_200_a076d17e202ed18ebdf8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb +size 165516 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_200_c0196720a45889bb2198.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_200_c0196720a45889bb2198.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018bd36397d571746795d1f4619cc8f4fc2766da --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_200_c0196720a45889bb2198.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8 +size 173236 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_200_c0f892b5fba78348af85.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_200_c0f892b5fba78348af85.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f296c5f3aec5b609e307e17bbeffa25264deb91 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_200_c0f892b5fba78348af85.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8 +size 222986 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_200_f5463a4ea6f68a39c5d0.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_200_f5463a4ea6f68a39c5d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..078934eedc27e8de1e5fed0272335f707868e553 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_200_f5463a4ea6f68a39c5d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395 +size 164630 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_200_f7412c9e170ffd299748.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_200_f7412c9e170ffd299748.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eaf7bba0c634cee305854ae2bbd756c90092d38 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_200_f7412c9e170ffd299748.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273 +size 227490 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_300_15ac5b765a401d035db1.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_300_15ac5b765a401d035db1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39d693cdae1da9a3a67729fa63439503876eafbc --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_300_15ac5b765a401d035db1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117 +size 190004 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_300_294040ac378ceaa34329.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_300_294040ac378ceaa34329.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23dc14931dfca40fe3f7d6b1fdefe0a1edca1ab9 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_300_294040ac378ceaa34329.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c +size 249372 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_300_6ff046418b5d41477989.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_300_6ff046418b5d41477989.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95ab423dcc64a40477b5d8f2e5d44351e2781109 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_300_6ff046418b5d41477989.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac +size 214001 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_300_80aab00fcfa85d5db98e.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_300_80aab00fcfa85d5db98e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09ef3a36f7d3e07e8b458916212891367b079325 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_300_80aab00fcfa85d5db98e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327 +size 182028 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_300_884b2d197a5511393680.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_300_884b2d197a5511393680.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..550cb7c37290fce2f4554ad8f26113320700bfa3 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_300_884b2d197a5511393680.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214 +size 170922 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_300_a076d17e202ed18ebdf8.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_300_a076d17e202ed18ebdf8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c59919de964bcf66a3cfe53f52bedd0a8b91df2e --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_300_a076d17e202ed18ebdf8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb +size 165516 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_300_c0196720a45889bb2198.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_300_c0196720a45889bb2198.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018bd36397d571746795d1f4619cc8f4fc2766da --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_300_c0196720a45889bb2198.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8 +size 173236 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_300_c0f892b5fba78348af85.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_300_c0f892b5fba78348af85.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f296c5f3aec5b609e307e17bbeffa25264deb91 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_300_c0f892b5fba78348af85.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8 +size 222986 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_300_f5463a4ea6f68a39c5d0.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_300_f5463a4ea6f68a39c5d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..078934eedc27e8de1e5fed0272335f707868e553 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_300_f5463a4ea6f68a39c5d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395 +size 164630 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_300_f7412c9e170ffd299748.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_300_f7412c9e170ffd299748.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eaf7bba0c634cee305854ae2bbd756c90092d38 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_300_f7412c9e170ffd299748.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273 +size 227490 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_400_15ac5b765a401d035db1.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_400_15ac5b765a401d035db1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39d693cdae1da9a3a67729fa63439503876eafbc --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_400_15ac5b765a401d035db1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117 +size 190004 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_400_294040ac378ceaa34329.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_400_294040ac378ceaa34329.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23dc14931dfca40fe3f7d6b1fdefe0a1edca1ab9 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_400_294040ac378ceaa34329.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c +size 249372 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_400_6ff046418b5d41477989.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_400_6ff046418b5d41477989.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95ab423dcc64a40477b5d8f2e5d44351e2781109 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_400_6ff046418b5d41477989.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac +size 214001 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_400_80aab00fcfa85d5db98e.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_400_80aab00fcfa85d5db98e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09ef3a36f7d3e07e8b458916212891367b079325 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_400_80aab00fcfa85d5db98e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327 +size 182028 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_400_884b2d197a5511393680.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_400_884b2d197a5511393680.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..550cb7c37290fce2f4554ad8f26113320700bfa3 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_400_884b2d197a5511393680.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214 +size 170922 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_400_a076d17e202ed18ebdf8.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_400_a076d17e202ed18ebdf8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c59919de964bcf66a3cfe53f52bedd0a8b91df2e --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_400_a076d17e202ed18ebdf8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb +size 165516 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_400_c0196720a45889bb2198.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_400_c0196720a45889bb2198.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018bd36397d571746795d1f4619cc8f4fc2766da --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_400_c0196720a45889bb2198.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8 +size 173236 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_400_c0f892b5fba78348af85.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_400_c0f892b5fba78348af85.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f296c5f3aec5b609e307e17bbeffa25264deb91 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_400_c0f892b5fba78348af85.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8 +size 222986 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_400_f5463a4ea6f68a39c5d0.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_400_f5463a4ea6f68a39c5d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..078934eedc27e8de1e5fed0272335f707868e553 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_400_f5463a4ea6f68a39c5d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395 +size 164630 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_400_f7412c9e170ffd299748.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_400_f7412c9e170ffd299748.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eaf7bba0c634cee305854ae2bbd756c90092d38 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_400_f7412c9e170ffd299748.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273 +size 227490 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_500_15ac5b765a401d035db1.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_500_15ac5b765a401d035db1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39d693cdae1da9a3a67729fa63439503876eafbc --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_500_15ac5b765a401d035db1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117 +size 190004 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_500_294040ac378ceaa34329.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_500_294040ac378ceaa34329.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23dc14931dfca40fe3f7d6b1fdefe0a1edca1ab9 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_500_294040ac378ceaa34329.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c +size 249372 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_500_6ff046418b5d41477989.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_500_6ff046418b5d41477989.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95ab423dcc64a40477b5d8f2e5d44351e2781109 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_500_6ff046418b5d41477989.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac +size 214001 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_500_80aab00fcfa85d5db98e.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_500_80aab00fcfa85d5db98e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09ef3a36f7d3e07e8b458916212891367b079325 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_500_80aab00fcfa85d5db98e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327 +size 182028 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_500_884b2d197a5511393680.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_500_884b2d197a5511393680.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..550cb7c37290fce2f4554ad8f26113320700bfa3 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_500_884b2d197a5511393680.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214 +size 170922 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_500_a076d17e202ed18ebdf8.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_500_a076d17e202ed18ebdf8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c59919de964bcf66a3cfe53f52bedd0a8b91df2e --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_500_a076d17e202ed18ebdf8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb +size 165516 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_500_c0196720a45889bb2198.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_500_c0196720a45889bb2198.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018bd36397d571746795d1f4619cc8f4fc2766da --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_500_c0196720a45889bb2198.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8 +size 173236 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_500_c0f892b5fba78348af85.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_500_c0f892b5fba78348af85.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f296c5f3aec5b609e307e17bbeffa25264deb91 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_500_c0f892b5fba78348af85.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8 +size 222986 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_500_f5463a4ea6f68a39c5d0.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_500_f5463a4ea6f68a39c5d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..078934eedc27e8de1e5fed0272335f707868e553 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_500_f5463a4ea6f68a39c5d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395 +size 164630 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_500_f7412c9e170ffd299748.mp4 b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_500_f7412c9e170ffd299748.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eaf7bba0c634cee305854ae2bbd756c90092d38 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/media/videos/val0/reconstructed_500_f7412c9e170ffd299748.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273 +size 227490 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/output.log b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..ab59ee9744cde9d29cff387e349d52c56f2e1c6a --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/output.log @@ -0,0 +1,12343 @@ +[Apr 30, 2026 - 22:12:06 | INFO | fastgen.callbacks.wandb:init_wandb:290 ] Wandb code upload disabled (set WANDB_UPLOAD_CODE=true to enable) +[Apr 30, 2026 - 22:12:06 | SUCCESS | fastgen.trainer:__init__:53 ] Callbacks initialized successfully +[Apr 30, 2026 - 22:12:06 | INFO | fastgen.trainer:__init__:57 ] Callback synchronization complete +[Apr 30, 2026 - 22:12:06 | INFO | fastgen.trainer:__init__:60 ] Initializing checkpointer... +[Apr 30, 2026 - 22:12:06 | SUCCESS | fastgen.trainer:__init__:65 ] Checkpointer initialized successfully +[Apr 30, 2026 - 22:12:06 | SUCCESS | __main__:main:33 ] Trainer initialized successfully +[Apr 30, 2026 - 22:12:06 | INFO | fastgen.trainer:run:77 ] Starting training +[Apr 30, 2026 - 22:12:06 | INFO | fastgen.trainer:run:80 ] Initializing callbacks and model ... +[Apr 30, 2026 - 22:12:06 | INFO | fastgen.utils.checkpointer:load:151 ] Loading model from /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_t769_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth +[Apr 30, 2026 - 22:12:10 | INFO | fastgen.utils.checkpointer:load:154 ] Loading the model_dict... +[Apr 30, 2026 - 22:12:22 | INFO | fastgen.utils.checkpointer:load:159 ] Model net, loading info: +[Apr 30, 2026 - 22:12:22 | INFO | fastgen.trainer:load_pretrained_ckpt:252 ] Loaded net model from net in /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_t769_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth at iteration 5000 +[Apr 30, 2026 - 22:12:22 | INFO | fastgen.trainer:load_pretrained_ckpt:262 ] Setting resume_iter for model to 5000. +[Apr 30, 2026 - 22:12:22 | INFO | fastgen.trainer:run:95 ] Starting model.on_train_begin ... +[Apr 30, 2026 - 22:13:05 | INFO | fastgen.methods.model:on_train_begin:315 ] Teacher check: add_teacher_to_fsdp_dict=True, fsdp_dict keys=['net', 'fake_score', 'teacher'], teacher in fsdp_dict=True +[Apr 30, 2026 - 22:13:05 | INFO | fastgen.trainer:run:99 ] model.on_train_begin completed +[Apr 30, 2026 - 22:13:05 | INFO | fastgen.trainer:run:110 ] Wrapping model into fsdp .. +[Apr 30, 2026 - 22:13:05 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:102 ] Fully sharding model with 4 ranks... +[Apr 30, 2026 - 22:13:05 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'net' (1.42B params)... +[Apr 30, 2026 - 22:13:05 | INFO | fastgen.networks.OmniAvatar.network_causal:fully_shard:2381 ] CausalOmniAvatarWan: keeping manual gradient checkpointing (not using apply_fsdp_checkpointing due to KV cache dynamics) +[Apr 30, 2026 - 22:13:07 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[Apr 30, 2026 - 22:13:07 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped net in 2.2s +[Apr 30, 2026 - 22:13:07 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'fake_score' (1.42B params)... +[Apr 30, 2026 - 22:13:07 | INFO | fastgen.networks.OmniAvatar.network:fully_shard:932 ] OmniAvatarWan: keeping manual gradient checkpointing (checkpoint_wrapper incompatible with inter-block audio injection) +[Apr 30, 2026 - 22:13:09 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[Apr 30, 2026 - 22:13:09 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped fake_score in 2.0s +[Apr 30, 2026 - 22:13:09 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'teacher' (14.29B params)... +[Apr 30, 2026 - 22:13:09 | INFO | fastgen.networks.OmniAvatar.network:fully_shard:932 ] OmniAvatarWan: keeping manual gradient checkpointing (checkpoint_wrapper incompatible with inter-block audio injection) +[Apr 30, 2026 - 22:13:21 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[Apr 30, 2026 - 22:13:21 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped teacher in 11.6s +[Apr 30, 2026 - 22:13:21 | INFO | fastgen.trainer:run:118 ] FSDP wrapping completed +[Apr 30, 2026 - 22:13:21 | INFO | fastgen.callbacks.ema:on_model_init_end:64 ] EMA ema is not enabled, skipping callback. +[Apr 30, 2026 - 22:13:21 | INFO | fastgen.trainer:run:133 ] Auto-Resume Details: None +[Apr 30, 2026 - 22:13:21 | INFO | fastgen.utils.basic_utils:set_random_seed:144 ] Using random seed 0. +[Apr 30, 2026 - 22:13:21 | INFO | fastgen.trainer:run:165 ] Instantiating dataloader... +[Apr 30, 2026 - 22:13:27 | INFO | fastgen.callbacks.wandb:on_dataloader_init_end:422 ] SKIP_GT_VAL_UPLOAD=1 — skipping GT val video upload +[Apr 30, 2026 - 22:13:27 | INFO | fastgen.callbacks.gpu_stats:on_train_begin:57 ] every_n to measure gpus stats: 1 +[Apr 30, 2026 - 22:13:27 | INFO | fastgen.callbacks.train_profiler:on_train_begin:54 ] every_n to profile trainer: 1 +[Apr 30, 2026 - 22:13:27 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] model (OmniAvatarSelfForcingReDMD) has 2842.77 M trainable and 17150.48 M total params (logical). +[Apr 30, 2026 - 22:13:27 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 711.11 M trainable and 4299.19 M total params LOCAL on rank 0. +[Apr 30, 2026 - 22:13:27 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 710.55 M trainable and 4297.39 M total params LOCAL on rank 1. +[Apr 30, 2026 - 22:13:27 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 710.55 M trainable and 4297.39 M total params LOCAL on rank 2. +[Apr 30, 2026 - 22:13:27 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 710.55 M trainable and 4297.39 M total params LOCAL on rank 3. +[Apr 30, 2026 - 22:13:27 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] net (CausalOmniAvatarWan) has 1421.38 M trainable and 1421.38 M total params (logical). +[Apr 30, 2026 - 22:13:27 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 355.56 M trainable and 355.56 M total params LOCAL on rank 0. +[Apr 30, 2026 - 22:13:27 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 1. +[Apr 30, 2026 - 22:13:27 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 2. +[Apr 30, 2026 - 22:13:27 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 3. +[Apr 30, 2026 - 22:13:27 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] fake_score (OmniAvatarWan) has 1421.38 M trainable and 1421.38 M total params (logical). +[Apr 30, 2026 - 22:13:27 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 355.56 M trainable and 355.56 M total params LOCAL on rank 0. +[Apr 30, 2026 - 22:13:27 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 1. +[Apr 30, 2026 - 22:13:27 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 2. +[Apr 30, 2026 - 22:13:27 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 355.28 M trainable and 355.28 M total params LOCAL on rank 3. +[Apr 30, 2026 - 22:13:27 | INFO | fastgen.trainer:run:174 ] iter_start: 0 +[Apr 30, 2026 - 22:14:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.379385 37.531902 37.251678 +peak_gpu_mem_gb 57.083078 57.083078 57.083078 +peak_gpu_mem_reserved_gb 59.166016 59.166016 59.166016 +util 80.750000 87.000000 77.000000 +[Apr 30, 2026 - 22:14:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 1 : data loading time 0.96 +[Apr 30, 2026 - 22:14:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 1 : avg forward pass time 10.42 +[Apr 30, 2026 - 22:14:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 1 : backward pass time 10.17 +[Apr 30, 2026 - 22:14:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 1 : optimizer step time 1.75 +[Apr 30, 2026 - 22:14:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 1-------------------- +[Apr 30, 2026 - 22:14:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[Apr 30, 2026 - 22:14:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[Apr 30, 2026 - 22:14:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 22:14:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 22:14:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.554615 37.707977 37.426521 +peak_gpu_mem_gb 59.730629 59.732192 59.730109 +peak_gpu_mem_reserved_gb 61.242676 61.244141 61.242188 +util 96.250000 100.000000 93.000000 +[Apr 30, 2026 - 22:14:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 2 : avg iteration time 46.40 seconds +[Apr 30, 2026 - 22:14:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 2 : data loading time 0.00 +[Apr 30, 2026 - 22:14:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 2 : avg forward pass time 12.24 +[Apr 30, 2026 - 22:14:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 2 : backward pass time 10.20 +[Apr 30, 2026 - 22:14:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 2 : optimizer step time 1.57 +[Apr 30, 2026 - 22:14:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 2-------------------- +[Apr 30, 2026 - 22:14:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[Apr 30, 2026 - 22:14:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[Apr 30, 2026 - 22:14:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 22:14:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 22:15:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.555037 37.708099 37.427116 +peak_gpu_mem_gb 59.730629 59.732192 59.730109 +peak_gpu_mem_reserved_gb 61.262207 61.263672 61.261719 +util 94.750000 99.000000 92.000000 +[Apr 30, 2026 - 22:15:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 3 : avg iteration time 49.18 seconds +[Apr 30, 2026 - 22:15:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 3 : data loading time 0.00 +[Apr 30, 2026 - 22:15:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 3 : avg forward pass time 13.63 +[Apr 30, 2026 - 22:15:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 3 : backward pass time 10.14 +[Apr 30, 2026 - 22:15:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 3 : optimizer step time 1.57 +[Apr 30, 2026 - 22:15:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 3-------------------- +[Apr 30, 2026 - 22:15:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0074 iter count: 1.0 +[Apr 30, 2026 - 22:15:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0074 iter count: 1.0 +[Apr 30, 2026 - 22:15:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 22:15:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 22:16:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.556276 37.710289 37.428329 +peak_gpu_mem_gb 59.730629 59.732192 59.730109 +peak_gpu_mem_reserved_gb 61.223145 61.224609 61.222656 +util 92.750000 99.000000 89.000000 +[Apr 30, 2026 - 22:16:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 4 : avg iteration time 43.77 seconds +[Apr 30, 2026 - 22:16:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 4 : data loading time 0.00 +[Apr 30, 2026 - 22:16:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 4 : avg forward pass time 10.94 +[Apr 30, 2026 - 22:16:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 4 : backward pass time 10.14 +[Apr 30, 2026 - 22:16:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 4 : optimizer step time 1.58 +[Apr 30, 2026 - 22:16:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 4-------------------- +[Apr 30, 2026 - 22:16:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[Apr 30, 2026 - 22:16:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[Apr 30, 2026 - 22:16:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 22:16:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 22:19:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.623182 37.776249 37.495129 +peak_gpu_mem_gb 127.093702 127.095264 127.093182 +peak_gpu_mem_reserved_gb 130.894043 130.910156 130.888672 +util 58.750000 68.000000 51.000000 +[Apr 30, 2026 - 22:19:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 5 : avg iteration time 174.56 seconds +[Apr 30, 2026 - 22:19:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 5 : data loading time 0.00 +[Apr 30, 2026 - 22:19:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 5 : avg forward pass time 78.56 +[Apr 30, 2026 - 22:19:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 5 : backward pass time 8.45 +[Apr 30, 2026 - 22:19:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 5 : optimizer step time 0.44 +[Apr 30, 2026 - 22:19:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 5-------------------- +[Apr 30, 2026 - 22:19:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 41679.4194 iter count: 1.0 +[Apr 30, 2026 - 22:19:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.8032 iter count: 1.0 +[Apr 30, 2026 - 22:19:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 41679.4194iter count: 1.0 +[Apr 30, 2026 - 22:19:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 22:19:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 3.3609iter count: 1.0 +[Apr 30, 2026 - 22:19:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.8474 iter count: 1.0 +[Apr 30, 2026 - 22:19:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.0039 iter count: 1.0 +[Apr 30, 2026 - 22:19:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 53578.6523iter count: 1.0 +[Apr 30, 2026 - 22:19:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 5.4455 iter count: 1.0 +[Apr 30, 2026 - 22:19:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 1212071.1250iter count: 1.0 +[Apr 30, 2026 - 22:19:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.8032iter count: 1.0 +[Apr 30, 2026 - 22:19:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.6668 iter count: 1.0 +[Apr 30, 2026 - 22:19:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 11312.4824iter count: 1.0 +[Apr 30, 2026 - 22:19:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 2.7134 iter count: 1.0 +[Apr 30, 2026 - 22:19:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 227.4311iter count: 1.0 +[Apr 30, 2026 - 22:19:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 6.1653 iter count: 1.0 +[Apr 30, 2026 - 22:19:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 226530.1094iter count: 1.0 +[Apr 30, 2026 - 22:19:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 1.8156 iter count: 1.0 +[Apr 30, 2026 - 22:19:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 37.7604 iter count: 1.0 +[Apr 30, 2026 - 22:19:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0077 iter count: 1.0 +[Apr 30, 2026 - 22:19:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 22:19:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 30, 2026 - 22:20:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.623285 37.776314 37.495285 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.838867 64.871094 64.828125 +util 95.250000 100.000000 92.000000 +[Apr 30, 2026 - 22:20:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 6 : avg iteration time 50.67 seconds +[Apr 30, 2026 - 22:20:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 6 : data loading time 0.00 +[Apr 30, 2026 - 22:20:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 6 : avg forward pass time 14.39 +[Apr 30, 2026 - 22:20:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 6 : backward pass time 10.15 +[Apr 30, 2026 - 22:20:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 6 : optimizer step time 1.57 +[Apr 30, 2026 - 22:20:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 6-------------------- +[Apr 30, 2026 - 22:20:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0075 iter count: 1.0 +[Apr 30, 2026 - 22:20:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0075 iter count: 1.0 +[Apr 30, 2026 - 22:20:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 22:20:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 22:21:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.623652 37.776417 37.495487 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.882812 65.281250 64.750000 +util 96.000000 100.000000 89.000000 +[Apr 30, 2026 - 22:21:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 7 : avg iteration time 48.91 seconds +[Apr 30, 2026 - 22:21:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 7 : data loading time 0.00 +[Apr 30, 2026 - 22:21:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 7 : avg forward pass time 13.56 +[Apr 30, 2026 - 22:21:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 7 : backward pass time 10.13 +[Apr 30, 2026 - 22:21:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 7 : optimizer step time 1.55 +[Apr 30, 2026 - 22:21:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 7-------------------- +[Apr 30, 2026 - 22:21:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0074 iter count: 1.0 +[Apr 30, 2026 - 22:21:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0074 iter count: 1.0 +[Apr 30, 2026 - 22:21:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 22:21:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 22:21:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.623817 37.776695 37.495579 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.794922 64.871094 64.769531 +util 91.500000 98.000000 88.000000 +[Apr 30, 2026 - 22:21:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 8 : avg iteration time 49.03 seconds +[Apr 30, 2026 - 22:21:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 8 : data loading time 0.00 +[Apr 30, 2026 - 22:21:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 8 : avg forward pass time 13.58 +[Apr 30, 2026 - 22:21:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 8 : backward pass time 10.15 +[Apr 30, 2026 - 22:21:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 8 : optimizer step time 1.57 +[Apr 30, 2026 - 22:21:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 8-------------------- +[Apr 30, 2026 - 22:21:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0069 iter count: 1.0 +[Apr 30, 2026 - 22:21:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0069 iter count: 1.0 +[Apr 30, 2026 - 22:21:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 22:21:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 22:22:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.624036 37.776722 37.496056 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.746094 64.851562 64.710938 +util 96.250000 100.000000 90.000000 +[Apr 30, 2026 - 22:22:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 9 : avg iteration time 43.78 seconds +[Apr 30, 2026 - 22:22:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 9 : data loading time 0.00 +[Apr 30, 2026 - 22:22:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 9 : avg forward pass time 10.97 +[Apr 30, 2026 - 22:22:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 9 : backward pass time 10.14 +[Apr 30, 2026 - 22:22:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 9 : optimizer step time 1.57 +[Apr 30, 2026 - 22:22:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 9-------------------- +[Apr 30, 2026 - 22:22:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0068 iter count: 1.0 +[Apr 30, 2026 - 22:22:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0068 iter count: 1.0 +[Apr 30, 2026 - 22:22:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 22:22:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 22:25:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.625375 37.777416 37.497887 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.575684 133.591797 133.570312 +util 60.750000 67.000000 57.000000 +[Apr 30, 2026 - 22:25:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 10 : avg iteration time 169.84 seconds +[Apr 30, 2026 - 22:25:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 10 : data loading time 0.00 +[Apr 30, 2026 - 22:25:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 10 : avg forward pass time 76.29 +[Apr 30, 2026 - 22:25:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 10 : backward pass time 8.40 +[Apr 30, 2026 - 22:25:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 10 : optimizer step time 0.39 +[Apr 30, 2026 - 22:25:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 10-------------------- +[Apr 30, 2026 - 22:25:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 24658.6973 iter count: 1.0 +[Apr 30, 2026 - 22:25:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7820 iter count: 1.0 +[Apr 30, 2026 - 22:25:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 24658.6973iter count: 1.0 +[Apr 30, 2026 - 22:25:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 22:25:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 3.9291iter count: 1.0 +[Apr 30, 2026 - 22:25:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.6259 iter count: 1.0 +[Apr 30, 2026 - 22:25:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 6.5627 iter count: 1.0 +[Apr 30, 2026 - 22:25:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 31588.9785iter count: 1.0 +[Apr 30, 2026 - 22:25:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 3.4970 iter count: 1.0 +[Apr 30, 2026 - 22:25:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 501526.6250iter count: 1.0 +[Apr 30, 2026 - 22:25:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7820iter count: 1.0 +[Apr 30, 2026 - 22:25:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 6.5627 iter count: 1.0 +[Apr 30, 2026 - 22:25:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 501526.6250iter count: 1.0 +[Apr 30, 2026 - 22:25:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 2.4047 iter count: 1.0 +[Apr 30, 2026 - 22:25:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 122.6546iter count: 1.0 +[Apr 30, 2026 - 22:25:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.1709 iter count: 1.0 +[Apr 30, 2026 - 22:25:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 4195.8457iter count: 1.0 +[Apr 30, 2026 - 22:25:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.8135 iter count: 1.0 +[Apr 30, 2026 - 22:25:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 112083.0234iter count: 1.0 +[Apr 30, 2026 - 22:25:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0069 iter count: 1.0 +[Apr 30, 2026 - 22:25:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 22:25:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 30, 2026 - 22:26:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.625494 37.777538 37.497936 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.868164 64.929688 64.847656 +util 90.750000 97.000000 87.000000 +[Apr 30, 2026 - 22:26:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 11 : avg iteration time 47.99 seconds +[Apr 30, 2026 - 22:26:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 11 : data loading time 0.00 +[Apr 30, 2026 - 22:26:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 11 : avg forward pass time 13.06 +[Apr 30, 2026 - 22:26:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 11 : backward pass time 10.13 +[Apr 30, 2026 - 22:26:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 11 : optimizer step time 1.57 +[Apr 30, 2026 - 22:26:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 11-------------------- +[Apr 30, 2026 - 22:26:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0076 iter count: 1.0 +[Apr 30, 2026 - 22:26:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0076 iter count: 1.0 +[Apr 30, 2026 - 22:26:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 22:26:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 22:27:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.625580 37.777546 37.498222 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.794922 64.871094 64.769531 +util 93.750000 99.000000 91.000000 +[Apr 30, 2026 - 22:27:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 12 : avg iteration time 49.03 seconds +[Apr 30, 2026 - 22:27:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 12 : data loading time 0.00 +[Apr 30, 2026 - 22:27:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 12 : avg forward pass time 13.59 +[Apr 30, 2026 - 22:27:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 12 : backward pass time 10.14 +[Apr 30, 2026 - 22:27:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 12 : optimizer step time 1.55 +[Apr 30, 2026 - 22:27:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 12-------------------- +[Apr 30, 2026 - 22:27:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0067 iter count: 1.0 +[Apr 30, 2026 - 22:27:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0067 iter count: 1.0 +[Apr 30, 2026 - 22:27:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 22:27:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 22:27:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.625610 37.777554 37.498322 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.775391 64.910156 64.730469 +util 94.000000 99.000000 91.000000 +[Apr 30, 2026 - 22:27:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 13 : avg iteration time 49.21 seconds +[Apr 30, 2026 - 22:27:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 13 : data loading time 0.00 +[Apr 30, 2026 - 22:27:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 13 : avg forward pass time 13.67 +[Apr 30, 2026 - 22:27:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 13 : backward pass time 10.13 +[Apr 30, 2026 - 22:27:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 13 : optimizer step time 1.58 +[Apr 30, 2026 - 22:27:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 13-------------------- +[Apr 30, 2026 - 22:27:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0064 iter count: 1.0 +[Apr 30, 2026 - 22:27:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0064 iter count: 1.0 +[Apr 30, 2026 - 22:27:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 22:27:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 22:28:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.625721 37.777596 37.498383 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.765625 64.871094 64.730469 +util 91.250000 98.000000 88.000000 +[Apr 30, 2026 - 22:28:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 14 : avg iteration time 46.56 seconds +[Apr 30, 2026 - 22:28:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 14 : data loading time 0.00 +[Apr 30, 2026 - 22:28:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 14 : avg forward pass time 12.36 +[Apr 30, 2026 - 22:28:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 14 : backward pass time 10.14 +[Apr 30, 2026 - 22:28:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 14 : optimizer step time 1.56 +[Apr 30, 2026 - 22:28:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 14-------------------- +[Apr 30, 2026 - 22:28:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0068 iter count: 1.0 +[Apr 30, 2026 - 22:28:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0068 iter count: 1.0 +[Apr 30, 2026 - 22:28:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 22:28:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 22:31:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.626495 37.778145 37.500622 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.619629 133.628906 133.591797 +util 60.500000 67.000000 56.000000 +[Apr 30, 2026 - 22:31:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 15 : avg iteration time 163.82 seconds +[Apr 30, 2026 - 22:31:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 15 : data loading time 0.00 +[Apr 30, 2026 - 22:31:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 15 : avg forward pass time 73.29 +[Apr 30, 2026 - 22:31:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 15 : backward pass time 8.43 +[Apr 30, 2026 - 22:31:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 15 : optimizer step time 0.40 +[Apr 30, 2026 - 22:31:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 15-------------------- +[Apr 30, 2026 - 22:31:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 38548.7671 iter count: 1.0 +[Apr 30, 2026 - 22:31:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7329 iter count: 1.0 +[Apr 30, 2026 - 22:31:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 38548.7671iter count: 1.0 +[Apr 30, 2026 - 22:31:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 22:31:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 3.3806iter count: 1.0 +[Apr 30, 2026 - 22:31:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.6103 iter count: 1.0 +[Apr 30, 2026 - 22:31:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 6.9534 iter count: 1.0 +[Apr 30, 2026 - 22:31:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 46397.4688iter count: 1.0 +[Apr 30, 2026 - 22:31:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 3.3891 iter count: 1.0 +[Apr 30, 2026 - 22:31:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 1095625.1250iter count: 1.0 +[Apr 30, 2026 - 22:31:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7329iter count: 1.0 +[Apr 30, 2026 - 22:31:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.7283 iter count: 1.0 +[Apr 30, 2026 - 22:31:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 1731.3776iter count: 1.0 +[Apr 30, 2026 - 22:31:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.4719 iter count: 1.0 +[Apr 30, 2026 - 22:31:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 7660.5723iter count: 1.0 +[Apr 30, 2026 - 22:31:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 2.5718 iter count: 1.0 +[Apr 30, 2026 - 22:31:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 171.3397iter count: 1.0 +[Apr 30, 2026 - 22:31:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 2.5273 iter count: 1.0 +[Apr 30, 2026 - 22:31:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 156.7487iter count: 1.0 +[Apr 30, 2026 - 22:31:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0064 iter count: 1.0 +[Apr 30, 2026 - 22:31:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 22:31:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 30, 2026 - 22:32:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.626553 37.778149 37.500744 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.912109 64.929688 64.906250 +util 98.250000 100.000000 93.000000 +[Apr 30, 2026 - 22:32:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 16 : avg iteration time 45.33 seconds +[Apr 30, 2026 - 22:32:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 16 : data loading time 0.00 +[Apr 30, 2026 - 22:32:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 16 : avg forward pass time 11.74 +[Apr 30, 2026 - 22:32:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 16 : backward pass time 10.13 +[Apr 30, 2026 - 22:32:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 16 : optimizer step time 1.58 +[Apr 30, 2026 - 22:32:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 16-------------------- +[Apr 30, 2026 - 22:32:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0064 iter count: 1.0 +[Apr 30, 2026 - 22:32:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0064 iter count: 1.0 +[Apr 30, 2026 - 22:32:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 22:32:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 22:32:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.626590 37.778248 37.500729 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.873047 64.949219 64.847656 +util 95.250000 100.000000 92.000000 +[Apr 30, 2026 - 22:32:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 17 : avg iteration time 46.31 seconds +[Apr 30, 2026 - 22:32:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 17 : data loading time 0.00 +[Apr 30, 2026 - 22:32:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 17 : avg forward pass time 12.22 +[Apr 30, 2026 - 22:32:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 17 : backward pass time 10.14 +[Apr 30, 2026 - 22:32:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 17 : optimizer step time 1.58 +[Apr 30, 2026 - 22:32:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 17-------------------- +[Apr 30, 2026 - 22:32:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0068 iter count: 1.0 +[Apr 30, 2026 - 22:32:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0068 iter count: 1.0 +[Apr 30, 2026 - 22:32:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 22:32:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 22:33:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.626830 37.778248 37.500755 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.790039 64.968750 64.730469 +util 97.500000 99.000000 93.000000 +[Apr 30, 2026 - 22:33:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 18 : avg iteration time 46.48 seconds +[Apr 30, 2026 - 22:33:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 18 : data loading time 0.00 +[Apr 30, 2026 - 22:33:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 18 : avg forward pass time 12.30 +[Apr 30, 2026 - 22:33:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 18 : backward pass time 10.16 +[Apr 30, 2026 - 22:33:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 18 : optimizer step time 1.59 +[Apr 30, 2026 - 22:33:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 18-------------------- +[Apr 30, 2026 - 22:33:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0063 iter count: 1.0 +[Apr 30, 2026 - 22:33:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0063 iter count: 1.0 +[Apr 30, 2026 - 22:33:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 22:33:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 22:34:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.626877 37.778297 37.500713 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.736328 64.812500 64.710938 +util 96.250000 99.000000 91.000000 +[Apr 30, 2026 - 22:34:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 19 : avg iteration time 46.37 seconds +[Apr 30, 2026 - 22:34:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 19 : data loading time 0.00 +[Apr 30, 2026 - 22:34:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 19 : avg forward pass time 12.26 +[Apr 30, 2026 - 22:34:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 19 : backward pass time 10.14 +[Apr 30, 2026 - 22:34:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 19 : optimizer step time 1.58 +[Apr 30, 2026 - 22:34:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 19-------------------- +[Apr 30, 2026 - 22:34:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0064 iter count: 1.0 +[Apr 30, 2026 - 22:34:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0064 iter count: 1.0 +[Apr 30, 2026 - 22:34:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 22:34:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 22:37:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.627129 37.778435 37.500980 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.653809 133.669922 133.648438 +util 67.500000 71.000000 62.000000 +[Apr 30, 2026 - 22:37:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 20 : avg iteration time 170.23 seconds +[Apr 30, 2026 - 22:37:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 20 : data loading time 0.00 +[Apr 30, 2026 - 22:37:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 20 : avg forward pass time 76.50 +[Apr 30, 2026 - 22:37:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 20 : backward pass time 8.40 +[Apr 30, 2026 - 22:37:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 20 : optimizer step time 0.40 +[Apr 30, 2026 - 22:37:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 20-------------------- +[Apr 30, 2026 - 22:37:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 45542.1138 iter count: 1.0 +[Apr 30, 2026 - 22:37:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.8635 iter count: 1.0 +[Apr 30, 2026 - 22:37:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 45542.1138iter count: 1.0 +[Apr 30, 2026 - 22:37:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 22:37:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 3.6287iter count: 1.0 +[Apr 30, 2026 - 22:37:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.3054 iter count: 1.0 +[Apr 30, 2026 - 22:37:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 6.8267 iter count: 1.0 +[Apr 30, 2026 - 22:37:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 56166.6328iter count: 1.0 +[Apr 30, 2026 - 22:37:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 13.6105iter count: 1.0 +[Apr 30, 2026 - 22:37:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 850350.7500iter count: 1.0 +[Apr 30, 2026 - 22:37:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.8635iter count: 1.0 +[Apr 30, 2026 - 22:37:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.3241 iter count: 1.0 +[Apr 30, 2026 - 22:37:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 42120.8555iter count: 1.0 +[Apr 30, 2026 - 22:37:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 2.0888 iter count: 1.0 +[Apr 30, 2026 - 22:37:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 65.2121 iter count: 1.0 +[Apr 30, 2026 - 22:37:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.4846 iter count: 1.0 +[Apr 30, 2026 - 22:37:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 7857.9443iter count: 1.0 +[Apr 30, 2026 - 22:37:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.8267 iter count: 1.0 +[Apr 30, 2026 - 22:37:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 850350.7500iter count: 1.0 +[Apr 30, 2026 - 22:37:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0070 iter count: 1.0 +[Apr 30, 2026 - 22:37:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 22:37:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 30, 2026 - 22:38:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.627194 37.778446 37.501087 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.702148 64.730469 64.617188 +util 95.250000 100.000000 88.000000 +[Apr 30, 2026 - 22:38:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 21 : avg iteration time 45.62 seconds +[Apr 30, 2026 - 22:38:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 21 : data loading time 0.00 +[Apr 30, 2026 - 22:38:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 21 : avg forward pass time 11.89 +[Apr 30, 2026 - 22:38:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 21 : backward pass time 10.13 +[Apr 30, 2026 - 22:38:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 21 : optimizer step time 1.58 +[Apr 30, 2026 - 22:38:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 21-------------------- +[Apr 30, 2026 - 22:38:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0061 iter count: 1.0 +[Apr 30, 2026 - 22:38:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0061 iter count: 1.0 +[Apr 30, 2026 - 22:38:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 22:38:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 22:38:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.627212 37.778450 37.501167 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.731445 64.769531 64.617188 +util 95.750000 100.000000 88.000000 +[Apr 30, 2026 - 22:38:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 22 : avg iteration time 46.40 seconds +[Apr 30, 2026 - 22:38:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 22 : data loading time 0.00 +[Apr 30, 2026 - 22:38:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 22 : avg forward pass time 12.26 +[Apr 30, 2026 - 22:38:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 22 : backward pass time 10.15 +[Apr 30, 2026 - 22:38:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 22 : optimizer step time 1.57 +[Apr 30, 2026 - 22:38:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 22-------------------- +[Apr 30, 2026 - 22:38:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0067 iter count: 1.0 +[Apr 30, 2026 - 22:38:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0067 iter count: 1.0 +[Apr 30, 2026 - 22:38:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 22:38:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 30, 2026 - 22:39:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.627348 37.778770 37.501339 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.775391 65.027344 64.691406 +util 94.750000 99.000000 88.000000 +[Apr 30, 2026 - 22:39:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 23 : avg iteration time 46.35 seconds +[Apr 30, 2026 - 22:39:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 23 : data loading time 0.00 +[Apr 30, 2026 - 22:39:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 23 : avg forward pass time 12.24 +[Apr 30, 2026 - 22:39:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 23 : backward pass time 10.15 +[Apr 30, 2026 - 22:39:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 23 : optimizer step time 1.59 +[Apr 30, 2026 - 22:39:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 23-------------------- +[Apr 30, 2026 - 22:39:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0059 iter count: 1.0 +[Apr 30, 2026 - 22:39:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0059 iter count: 1.0 +[Apr 30, 2026 - 22:39:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 22:39:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 22:40:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.627416 37.778816 37.501339 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.843750 65.066406 64.769531 +util 94.750000 99.000000 92.000000 +[Apr 30, 2026 - 22:40:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 24 : avg iteration time 46.34 seconds +[Apr 30, 2026 - 22:40:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 24 : data loading time 0.00 +[Apr 30, 2026 - 22:40:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 24 : avg forward pass time 12.23 +[Apr 30, 2026 - 22:40:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 24 : backward pass time 10.15 +[Apr 30, 2026 - 22:40:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 24 : optimizer step time 1.58 +[Apr 30, 2026 - 22:40:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 24-------------------- +[Apr 30, 2026 - 22:40:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0065 iter count: 1.0 +[Apr 30, 2026 - 22:40:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0065 iter count: 1.0 +[Apr 30, 2026 - 22:40:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 22:40:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 22:43:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.627483 37.778862 37.501472 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.653809 133.667969 133.611328 +util 61.250000 68.000000 57.000000 +[Apr 30, 2026 - 22:43:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 25 : avg iteration time 163.96 seconds +[Apr 30, 2026 - 22:43:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 25 : data loading time 0.00 +[Apr 30, 2026 - 22:43:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 25 : avg forward pass time 73.36 +[Apr 30, 2026 - 22:43:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 25 : backward pass time 8.45 +[Apr 30, 2026 - 22:43:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 25 : optimizer step time 0.39 +[Apr 30, 2026 - 22:43:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 25-------------------- +[Apr 30, 2026 - 22:43:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 56198.5740 iter count: 1.0 +[Apr 30, 2026 - 22:43:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7803 iter count: 1.0 +[Apr 30, 2026 - 22:43:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 56198.5740iter count: 1.0 +[Apr 30, 2026 - 22:43:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 22:43:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 3.2383iter count: 1.0 +[Apr 30, 2026 - 22:43:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.5383 iter count: 1.0 +[Apr 30, 2026 - 22:43:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.0252 iter count: 1.0 +[Apr 30, 2026 - 22:43:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 68084.3125iter count: 1.0 +[Apr 30, 2026 - 22:43:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 2.9347 iter count: 1.0 +[Apr 30, 2026 - 22:43:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 1264873.5000iter count: 1.0 +[Apr 30, 2026 - 22:43:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7803iter count: 1.0 +[Apr 30, 2026 - 22:43:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 1.8680 iter count: 1.0 +[Apr 30, 2026 - 22:43:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 41.9303 iter count: 1.0 +[Apr 30, 2026 - 22:43:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 0.8068 iter count: 1.0 +[Apr 30, 2026 - 22:43:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 5.0211 iter count: 1.0 +[Apr 30, 2026 - 22:43:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.7842 iter count: 1.0 +[Apr 30, 2026 - 22:43:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 1936.2126iter count: 1.0 +[Apr 30, 2026 - 22:43:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.0711 iter count: 1.0 +[Apr 30, 2026 - 22:43:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 187619.7969iter count: 1.0 +[Apr 30, 2026 - 22:43:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0061 iter count: 1.0 +[Apr 30, 2026 - 22:43:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 22:43:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 30, 2026 - 22:43:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.627495 37.778854 37.501476 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.848633 64.867188 64.792969 +util 90.250000 97.000000 87.000000 +[Apr 30, 2026 - 22:43:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 26 : avg iteration time 45.37 seconds +[Apr 30, 2026 - 22:43:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 26 : data loading time 0.00 +[Apr 30, 2026 - 22:43:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 26 : avg forward pass time 11.74 +[Apr 30, 2026 - 22:43:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 26 : backward pass time 10.15 +[Apr 30, 2026 - 22:43:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 26 : optimizer step time 1.58 +[Apr 30, 2026 - 22:43:54 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 26-------------------- +[Apr 30, 2026 - 22:43:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0067 iter count: 1.0 +[Apr 30, 2026 - 22:43:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0067 iter count: 1.0 +[Apr 30, 2026 - 22:43:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 22:43:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 22:44:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.627533 37.778854 37.501530 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.731445 64.750000 64.675781 +util 93.000000 99.000000 90.000000 +[Apr 30, 2026 - 22:44:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 27 : avg iteration time 49.23 seconds +[Apr 30, 2026 - 22:44:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 27 : data loading time 0.00 +[Apr 30, 2026 - 22:44:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 27 : avg forward pass time 13.68 +[Apr 30, 2026 - 22:44:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 27 : backward pass time 10.14 +[Apr 30, 2026 - 22:44:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 27 : optimizer step time 1.58 +[Apr 30, 2026 - 22:44:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 27-------------------- +[Apr 30, 2026 - 22:44:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0066 iter count: 1.0 +[Apr 30, 2026 - 22:44:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0066 iter count: 1.0 +[Apr 30, 2026 - 22:44:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 22:44:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 22:45:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.627492 37.778625 37.501587 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 65.883789 66.273438 64.714844 +util 92.750000 99.000000 89.000000 +[Apr 30, 2026 - 22:45:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 28 : avg iteration time 48.99 seconds +[Apr 30, 2026 - 22:45:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 28 : data loading time 0.00 +[Apr 30, 2026 - 22:45:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 28 : avg forward pass time 13.56 +[Apr 30, 2026 - 22:45:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 28 : backward pass time 10.15 +[Apr 30, 2026 - 22:45:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 28 : optimizer step time 1.58 +[Apr 30, 2026 - 22:45:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 28-------------------- +[Apr 30, 2026 - 22:45:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0062 iter count: 1.0 +[Apr 30, 2026 - 22:45:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0062 iter count: 1.0 +[Apr 30, 2026 - 22:45:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 22:45:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 22:46:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.627552 37.778648 37.501633 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.721680 64.753906 64.710938 +util 97.750000 100.000000 93.000000 +[Apr 30, 2026 - 22:46:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 29 : avg iteration time 46.33 seconds +[Apr 30, 2026 - 22:46:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 29 : data loading time 0.00 +[Apr 30, 2026 - 22:46:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 29 : avg forward pass time 12.24 +[Apr 30, 2026 - 22:46:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 29 : backward pass time 10.15 +[Apr 30, 2026 - 22:46:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 29 : optimizer step time 1.56 +[Apr 30, 2026 - 22:46:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 29-------------------- +[Apr 30, 2026 - 22:46:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0061 iter count: 1.0 +[Apr 30, 2026 - 22:46:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0061 iter count: 1.0 +[Apr 30, 2026 - 22:46:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 22:46:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 22:49:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.627697 37.778782 37.501801 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.623047 133.669922 133.607422 +util 62.750000 69.000000 59.000000 +[Apr 30, 2026 - 22:49:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 30 : avg iteration time 171.88 seconds +[Apr 30, 2026 - 22:49:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 30 : data loading time 0.00 +[Apr 30, 2026 - 22:49:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 30 : avg forward pass time 77.35 +[Apr 30, 2026 - 22:49:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 30 : backward pass time 8.41 +[Apr 30, 2026 - 22:49:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 30 : optimizer step time 0.39 +[Apr 30, 2026 - 22:49:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 30-------------------- +[Apr 30, 2026 - 22:49:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 1251492.6888 iter count: 1.0 +[Apr 30, 2026 - 22:49:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.8256 iter count: 1.0 +[Apr 30, 2026 - 22:49:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 1251492.6888iter count: 1.0 +[Apr 30, 2026 - 22:49:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 22:49:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 3.9054iter count: 1.0 +[Apr 30, 2026 - 22:49:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.4029 iter count: 1.0 +[Apr 30, 2026 - 22:49:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.8701 iter count: 1.0 +[Apr 30, 2026 - 22:49:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 1605952.6250iter count: 1.0 +[Apr 30, 2026 - 22:49:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 16.5414iter count: 1.0 +[Apr 30, 2026 - 22:49:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 50638320.0000iter count: 1.0 +[Apr 30, 2026 - 22:49:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.8256iter count: 1.0 +[Apr 30, 2026 - 22:49:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.0383 iter count: 1.0 +[Apr 30, 2026 - 22:49:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 23778.9785iter count: 1.0 +[Apr 30, 2026 - 22:49:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.4783 iter count: 1.0 +[Apr 30, 2026 - 22:49:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 7758.5195iter count: 1.0 +[Apr 30, 2026 - 22:49:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 2.6038 iter count: 1.0 +[Apr 30, 2026 - 22:49:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 182.6523iter count: 1.0 +[Apr 30, 2026 - 22:49:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.9153 iter count: 1.0 +[Apr 30, 2026 - 22:49:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 18593.8359iter count: 1.0 +[Apr 30, 2026 - 22:49:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0065 iter count: 1.0 +[Apr 30, 2026 - 22:49:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 22:49:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 30, 2026 - 22:49:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.627720 37.778790 37.501812 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.926758 64.945312 64.871094 +util 97.250000 99.000000 92.000000 +[Apr 30, 2026 - 22:49:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 31 : avg iteration time 45.49 seconds +[Apr 30, 2026 - 22:49:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 31 : data loading time 0.00 +[Apr 30, 2026 - 22:49:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 31 : avg forward pass time 11.83 +[Apr 30, 2026 - 22:49:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 31 : backward pass time 10.13 +[Apr 30, 2026 - 22:49:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 31 : optimizer step time 1.57 +[Apr 30, 2026 - 22:49:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 31-------------------- +[Apr 30, 2026 - 22:49:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0077 iter count: 1.0 +[Apr 30, 2026 - 22:49:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0077 iter count: 1.0 +[Apr 30, 2026 - 22:49:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 22:49:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 22:50:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.627746 37.778793 37.501839 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.877930 64.886719 64.851562 +util 95.250000 100.000000 92.000000 +[Apr 30, 2026 - 22:50:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 32 : avg iteration time 46.33 seconds +[Apr 30, 2026 - 22:50:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 32 : data loading time 0.00 +[Apr 30, 2026 - 22:50:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 32 : avg forward pass time 12.23 +[Apr 30, 2026 - 22:50:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 32 : backward pass time 10.16 +[Apr 30, 2026 - 22:50:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 32 : optimizer step time 1.58 +[Apr 30, 2026 - 22:50:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 32-------------------- +[Apr 30, 2026 - 22:50:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0063 iter count: 1.0 +[Apr 30, 2026 - 22:50:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0063 iter count: 1.0 +[Apr 30, 2026 - 22:50:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 22:50:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 22:51:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.627788 37.778797 37.501846 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.882812 64.886719 64.871094 +util 95.250000 99.000000 93.000000 +[Apr 30, 2026 - 22:51:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 33 : avg iteration time 43.61 seconds +[Apr 30, 2026 - 22:51:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 33 : data loading time 0.00 +[Apr 30, 2026 - 22:51:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 33 : avg forward pass time 10.88 +[Apr 30, 2026 - 22:51:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 33 : backward pass time 10.14 +[Apr 30, 2026 - 22:51:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 33 : optimizer step time 1.56 +[Apr 30, 2026 - 22:51:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 33-------------------- +[Apr 30, 2026 - 22:51:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0058 iter count: 1.0 +[Apr 30, 2026 - 22:51:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0058 iter count: 1.0 +[Apr 30, 2026 - 22:51:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 22:51:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 22:52:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.627810 37.778797 37.501823 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.897461 64.906250 64.871094 +util 95.250000 99.000000 93.000000 +[Apr 30, 2026 - 22:52:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 34 : avg iteration time 49.00 seconds +[Apr 30, 2026 - 22:52:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 34 : data loading time 0.00 +[Apr 30, 2026 - 22:52:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 34 : avg forward pass time 13.59 +[Apr 30, 2026 - 22:52:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 34 : backward pass time 10.11 +[Apr 30, 2026 - 22:52:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 34 : optimizer step time 1.58 +[Apr 30, 2026 - 22:52:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 34-------------------- +[Apr 30, 2026 - 22:52:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0064 iter count: 1.0 +[Apr 30, 2026 - 22:52:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0064 iter count: 1.0 +[Apr 30, 2026 - 22:52:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 22:52:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 22:55:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.627593 37.777863 37.501858 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.613281 133.626953 133.572266 +util 64.500000 74.000000 56.000000 +[Apr 30, 2026 - 22:55:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 35 : avg iteration time 167.13 seconds +[Apr 30, 2026 - 22:55:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 35 : data loading time 0.00 +[Apr 30, 2026 - 22:55:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 35 : avg forward pass time 74.94 +[Apr 30, 2026 - 22:55:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 35 : backward pass time 8.46 +[Apr 30, 2026 - 22:55:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 35 : optimizer step time 0.39 +[Apr 30, 2026 - 22:55:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 35-------------------- +[Apr 30, 2026 - 22:55:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 49944.1868 iter count: 1.0 +[Apr 30, 2026 - 22:55:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7492 iter count: 1.0 +[Apr 30, 2026 - 22:55:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 49944.1868iter count: 1.0 +[Apr 30, 2026 - 22:55:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 22:55:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 3.8015iter count: 1.0 +[Apr 30, 2026 - 22:55:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.9355 iter count: 1.0 +[Apr 30, 2026 - 22:55:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 6.8365 iter count: 1.0 +[Apr 30, 2026 - 22:55:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 61721.3555iter count: 1.0 +[Apr 30, 2026 - 22:55:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 6.4950 iter count: 1.0 +[Apr 30, 2026 - 22:55:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 867173.6875iter count: 1.0 +[Apr 30, 2026 - 22:55:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7492iter count: 1.0 +[Apr 30, 2026 - 22:55:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.9962 iter count: 1.0 +[Apr 30, 2026 - 22:55:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 2958.3354iter count: 1.0 +[Apr 30, 2026 - 22:55:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 0.9752 iter count: 1.0 +[Apr 30, 2026 - 22:55:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 7.0311 iter count: 1.0 +[Apr 30, 2026 - 22:55:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.2453 iter count: 1.0 +[Apr 30, 2026 - 22:55:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 35977.8867iter count: 1.0 +[Apr 30, 2026 - 22:55:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 2.7113 iter count: 1.0 +[Apr 30, 2026 - 22:55:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 226.4753iter count: 1.0 +[Apr 30, 2026 - 22:55:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0061 iter count: 1.0 +[Apr 30, 2026 - 22:55:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 22:55:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 30, 2026 - 22:55:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.627755 37.778378 37.501930 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.897461 64.929688 64.886719 +util 96.500000 99.000000 91.000000 +[Apr 30, 2026 - 22:55:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 36 : avg iteration time 48.12 seconds +[Apr 30, 2026 - 22:55:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 36 : data loading time 0.00 +[Apr 30, 2026 - 22:55:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 36 : avg forward pass time 13.12 +[Apr 30, 2026 - 22:55:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 36 : backward pass time 10.15 +[Apr 30, 2026 - 22:55:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 36 : optimizer step time 1.58 +[Apr 30, 2026 - 22:55:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 36-------------------- +[Apr 30, 2026 - 22:55:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0081 iter count: 1.0 +[Apr 30, 2026 - 22:55:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[Apr 30, 2026 - 22:55:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 22:55:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 22:56:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.627776 37.778366 37.501949 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.843750 64.847656 64.832031 +util 96.250000 100.000000 89.000000 +[Apr 30, 2026 - 22:56:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 37 : avg iteration time 43.67 seconds +[Apr 30, 2026 - 22:56:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 37 : data loading time 0.00 +[Apr 30, 2026 - 22:56:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 37 : avg forward pass time 10.91 +[Apr 30, 2026 - 22:56:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 37 : backward pass time 10.15 +[Apr 30, 2026 - 22:56:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 37 : optimizer step time 1.57 +[Apr 30, 2026 - 22:56:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 37-------------------- +[Apr 30, 2026 - 22:56:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0061 iter count: 1.0 +[Apr 30, 2026 - 22:56:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0061 iter count: 1.0 +[Apr 30, 2026 - 22:56:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 22:56:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 22:57:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.627808 37.778389 37.502007 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.819336 64.910156 64.789062 +util 90.750000 97.000000 87.000000 +[Apr 30, 2026 - 22:57:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 38 : avg iteration time 49.02 seconds +[Apr 30, 2026 - 22:57:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 38 : data loading time 0.00 +[Apr 30, 2026 - 22:57:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 38 : avg forward pass time 13.58 +[Apr 30, 2026 - 22:57:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 38 : backward pass time 10.13 +[Apr 30, 2026 - 22:57:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 38 : optimizer step time 1.58 +[Apr 30, 2026 - 22:57:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 38-------------------- +[Apr 30, 2026 - 22:57:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0065 iter count: 1.0 +[Apr 30, 2026 - 22:57:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0065 iter count: 1.0 +[Apr 30, 2026 - 22:57:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 22:57:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 22:58:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.627947 37.778954 37.501999 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.892578 64.910156 64.886719 +util 95.750000 100.000000 88.000000 +[Apr 30, 2026 - 22:58:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 39 : avg iteration time 46.37 seconds +[Apr 30, 2026 - 22:58:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 39 : data loading time 0.00 +[Apr 30, 2026 - 22:58:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 39 : avg forward pass time 12.26 +[Apr 30, 2026 - 22:58:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 39 : backward pass time 10.15 +[Apr 30, 2026 - 22:58:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 39 : optimizer step time 1.57 +[Apr 30, 2026 - 22:58:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 39-------------------- +[Apr 30, 2026 - 22:58:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0069 iter count: 1.0 +[Apr 30, 2026 - 22:58:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0069 iter count: 1.0 +[Apr 30, 2026 - 22:58:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 22:58:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 23:00:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.627757 37.778236 37.502007 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.623047 133.646484 133.552734 +util 69.500000 74.000000 64.000000 +[Apr 30, 2026 - 23:00:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 40 : avg iteration time 167.00 seconds +[Apr 30, 2026 - 23:00:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 40 : data loading time 0.00 +[Apr 30, 2026 - 23:00:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 40 : avg forward pass time 74.91 +[Apr 30, 2026 - 23:00:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 40 : backward pass time 8.38 +[Apr 30, 2026 - 23:00:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 40 : optimizer step time 0.39 +[Apr 30, 2026 - 23:00:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 40-------------------- +[Apr 30, 2026 - 23:00:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 168264.9491 iter count: 1.0 +[Apr 30, 2026 - 23:00:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7750 iter count: 1.0 +[Apr 30, 2026 - 23:00:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 168264.9491iter count: 1.0 +[Apr 30, 2026 - 23:00:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:00:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.0115iter count: 1.0 +[Apr 30, 2026 - 23:00:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.7132 iter count: 1.0 +[Apr 30, 2026 - 23:00:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.3634 iter count: 1.0 +[Apr 30, 2026 - 23:00:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 179155.6719iter count: 1.0 +[Apr 30, 2026 - 23:00:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 4.1638 iter count: 1.0 +[Apr 30, 2026 - 23:00:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 2487717.7500iter count: 1.0 +[Apr 30, 2026 - 23:00:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7750iter count: 1.0 +[Apr 30, 2026 - 23:00:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.1184 iter count: 1.0 +[Apr 30, 2026 - 23:00:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 511.1708iter count: 1.0 +[Apr 30, 2026 - 23:00:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.4360 iter count: 1.0 +[Apr 30, 2026 - 23:00:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 964.8872iter count: 1.0 +[Apr 30, 2026 - 23:00:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 1.9033 iter count: 1.0 +[Apr 30, 2026 - 23:00:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 44.9963 iter count: 1.0 +[Apr 30, 2026 - 23:00:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.0745 iter count: 1.0 +[Apr 30, 2026 - 23:00:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 3460.2280iter count: 1.0 +[Apr 30, 2026 - 23:00:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0056 iter count: 1.0 +[Apr 30, 2026 - 23:00:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:00:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 30, 2026 - 23:01:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.627886 37.778736 37.502007 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.824219 64.906250 64.578125 +util 96.250000 100.000000 94.000000 +[Apr 30, 2026 - 23:01:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 41 : avg iteration time 50.71 seconds +[Apr 30, 2026 - 23:01:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 41 : data loading time 0.00 +[Apr 30, 2026 - 23:01:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 41 : avg forward pass time 14.42 +[Apr 30, 2026 - 23:01:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 41 : backward pass time 10.14 +[Apr 30, 2026 - 23:01:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 41 : optimizer step time 1.58 +[Apr 30, 2026 - 23:01:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 41-------------------- +[Apr 30, 2026 - 23:01:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0072 iter count: 1.0 +[Apr 30, 2026 - 23:01:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0072 iter count: 1.0 +[Apr 30, 2026 - 23:01:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:01:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 23:02:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.627947 37.778858 37.502007 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.775391 64.847656 64.558594 +util 96.000000 99.000000 90.000000 +[Apr 30, 2026 - 23:02:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 42 : avg iteration time 46.44 seconds +[Apr 30, 2026 - 23:02:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 42 : data loading time 0.00 +[Apr 30, 2026 - 23:02:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 42 : avg forward pass time 12.29 +[Apr 30, 2026 - 23:02:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 42 : backward pass time 10.15 +[Apr 30, 2026 - 23:02:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 42 : optimizer step time 1.56 +[Apr 30, 2026 - 23:02:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 42-------------------- +[Apr 30, 2026 - 23:02:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0068 iter count: 1.0 +[Apr 30, 2026 - 23:02:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0068 iter count: 1.0 +[Apr 30, 2026 - 23:02:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:02:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 23:03:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.627951 37.778870 37.502029 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.951172 65.027344 64.925781 +util 93.750000 99.000000 89.000000 +[Apr 30, 2026 - 23:03:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 43 : avg iteration time 43.88 seconds +[Apr 30, 2026 - 23:03:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 43 : data loading time 0.00 +[Apr 30, 2026 - 23:03:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 43 : avg forward pass time 11.02 +[Apr 30, 2026 - 23:03:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 43 : backward pass time 10.15 +[Apr 30, 2026 - 23:03:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 43 : optimizer step time 1.56 +[Apr 30, 2026 - 23:03:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 43-------------------- +[Apr 30, 2026 - 23:03:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0059 iter count: 1.0 +[Apr 30, 2026 - 23:03:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0059 iter count: 1.0 +[Apr 30, 2026 - 23:03:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:03:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 23:04:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.627999 37.779102 37.502007 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.794922 64.886719 64.519531 +util 98.000000 100.000000 94.000000 +[Apr 30, 2026 - 23:04:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 44 : avg iteration time 46.37 seconds +[Apr 30, 2026 - 23:04:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 44 : data loading time 0.00 +[Apr 30, 2026 - 23:04:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 44 : avg forward pass time 12.27 +[Apr 30, 2026 - 23:04:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 44 : backward pass time 10.13 +[Apr 30, 2026 - 23:04:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 44 : optimizer step time 1.57 +[Apr 30, 2026 - 23:04:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 44-------------------- +[Apr 30, 2026 - 23:04:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0070 iter count: 1.0 +[Apr 30, 2026 - 23:04:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0070 iter count: 1.0 +[Apr 30, 2026 - 23:04:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:04:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 23:06:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.627747 37.777966 37.502056 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.603516 133.650391 133.587891 +util 66.000000 70.000000 61.000000 +[Apr 30, 2026 - 23:06:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 45 : avg iteration time 166.96 seconds +[Apr 30, 2026 - 23:06:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 45 : data loading time 0.00 +[Apr 30, 2026 - 23:06:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 45 : avg forward pass time 74.84 +[Apr 30, 2026 - 23:06:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 45 : backward pass time 8.44 +[Apr 30, 2026 - 23:06:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 45 : optimizer step time 0.39 +[Apr 30, 2026 - 23:06:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 45-------------------- +[Apr 30, 2026 - 23:06:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 110116.8833 iter count: 1.0 +[Apr 30, 2026 - 23:06:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7646 iter count: 1.0 +[Apr 30, 2026 - 23:06:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 110116.8833iter count: 1.0 +[Apr 30, 2026 - 23:06:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:06:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.5245iter count: 1.0 +[Apr 30, 2026 - 23:06:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.9842 iter count: 1.0 +[Apr 30, 2026 - 23:06:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.2629 iter count: 1.0 +[Apr 30, 2026 - 23:06:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 163271.9844iter count: 1.0 +[Apr 30, 2026 - 23:06:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 52.9041iter count: 1.0 +[Apr 30, 2026 - 23:06:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 2034393.3750iter count: 1.0 +[Apr 30, 2026 - 23:06:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7646iter count: 1.0 +[Apr 30, 2026 - 23:06:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 6.0693 iter count: 1.0 +[Apr 30, 2026 - 23:06:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 186953.7656iter count: 1.0 +[Apr 30, 2026 - 23:06:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 2.7515 iter count: 1.0 +[Apr 30, 2026 - 23:06:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 245.4206iter count: 1.0 +[Apr 30, 2026 - 23:06:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.3535 iter count: 1.0 +[Apr 30, 2026 - 23:06:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 44665.9336iter count: 1.0 +[Apr 30, 2026 - 23:06:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.0119 iter count: 1.0 +[Apr 30, 2026 - 23:06:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 166673.8594iter count: 1.0 +[Apr 30, 2026 - 23:06:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0062 iter count: 1.0 +[Apr 30, 2026 - 23:06:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:06:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 30, 2026 - 23:07:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.627894 37.778526 37.502060 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.829102 64.906250 64.597656 +util 92.250000 99.000000 89.000000 +[Apr 30, 2026 - 23:07:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 46 : avg iteration time 50.70 seconds +[Apr 30, 2026 - 23:07:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 46 : data loading time 0.00 +[Apr 30, 2026 - 23:07:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 46 : avg forward pass time 14.43 +[Apr 30, 2026 - 23:07:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 46 : backward pass time 10.13 +[Apr 30, 2026 - 23:07:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 46 : optimizer step time 1.56 +[Apr 30, 2026 - 23:07:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 46-------------------- +[Apr 30, 2026 - 23:07:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0072 iter count: 1.0 +[Apr 30, 2026 - 23:07:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0072 iter count: 1.0 +[Apr 30, 2026 - 23:07:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:07:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 23:08:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.628032 37.779068 37.502090 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.794922 64.847656 64.636719 +util 96.500000 99.000000 91.000000 +[Apr 30, 2026 - 23:08:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 47 : avg iteration time 46.31 seconds +[Apr 30, 2026 - 23:08:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 47 : data loading time 0.00 +[Apr 30, 2026 - 23:08:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 47 : avg forward pass time 12.21 +[Apr 30, 2026 - 23:08:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 47 : backward pass time 10.15 +[Apr 30, 2026 - 23:08:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 47 : optimizer step time 1.58 +[Apr 30, 2026 - 23:08:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 47-------------------- +[Apr 30, 2026 - 23:08:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0075 iter count: 1.0 +[Apr 30, 2026 - 23:08:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0075 iter count: 1.0 +[Apr 30, 2026 - 23:08:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:08:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 23:09:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.628036 37.779068 37.502110 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.853516 64.929688 64.828125 +util 93.000000 99.000000 90.000000 +[Apr 30, 2026 - 23:09:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 48 : avg iteration time 49.10 seconds +[Apr 30, 2026 - 23:09:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 48 : data loading time 0.00 +[Apr 30, 2026 - 23:09:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 48 : avg forward pass time 13.62 +[Apr 30, 2026 - 23:09:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 48 : backward pass time 10.15 +[Apr 30, 2026 - 23:09:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 48 : optimizer step time 1.56 +[Apr 30, 2026 - 23:09:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 48-------------------- +[Apr 30, 2026 - 23:09:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0075 iter count: 1.0 +[Apr 30, 2026 - 23:09:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0075 iter count: 1.0 +[Apr 30, 2026 - 23:09:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:09:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 23:10:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.628160 37.779564 37.502087 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.804688 64.968750 64.750000 +util 95.250000 99.000000 93.000000 +[Apr 30, 2026 - 23:10:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 49 : avg iteration time 49.04 seconds +[Apr 30, 2026 - 23:10:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 49 : data loading time 0.00 +[Apr 30, 2026 - 23:10:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 49 : avg forward pass time 13.57 +[Apr 30, 2026 - 23:10:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 49 : backward pass time 10.13 +[Apr 30, 2026 - 23:10:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 49 : optimizer step time 1.58 +[Apr 30, 2026 - 23:10:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 49-------------------- +[Apr 30, 2026 - 23:10:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0072 iter count: 1.0 +[Apr 30, 2026 - 23:10:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0072 iter count: 1.0 +[Apr 30, 2026 - 23:10:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:10:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 23:12:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.627951 37.778652 37.502094 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.613281 133.626953 133.572266 +util 65.750000 73.000000 61.000000 +[Apr 30, 2026 - 23:12:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 50 : avg iteration time 164.03 seconds +[Apr 30, 2026 - 23:12:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 50 : data loading time 0.00 +[Apr 30, 2026 - 23:12:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 50 : avg forward pass time 73.38 +[Apr 30, 2026 - 23:12:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 50 : backward pass time 8.41 +[Apr 30, 2026 - 23:12:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 50 : optimizer step time 0.41 +[Apr 30, 2026 - 23:12:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 50-------------------- +[Apr 30, 2026 - 23:12:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 159686.1792 iter count: 1.0 +[Apr 30, 2026 - 23:12:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7751 iter count: 1.0 +[Apr 30, 2026 - 23:12:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 159686.1792iter count: 1.0 +[Apr 30, 2026 - 23:12:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:12:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.6381iter count: 1.0 +[Apr 30, 2026 - 23:12:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.9018 iter count: 1.0 +[Apr 30, 2026 - 23:12:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.5546 iter count: 1.0 +[Apr 30, 2026 - 23:12:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 232316.1406iter count: 1.0 +[Apr 30, 2026 - 23:12:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 44.8616iter count: 1.0 +[Apr 30, 2026 - 23:12:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 3646407.2500iter count: 1.0 +[Apr 30, 2026 - 23:12:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7751iter count: 1.0 +[Apr 30, 2026 - 23:12:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 7.0391 iter count: 1.0 +[Apr 30, 2026 - 23:12:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 1300471.6250iter count: 1.0 +[Apr 30, 2026 - 23:12:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 2.0378 iter count: 1.0 +[Apr 30, 2026 - 23:12:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 58.8896 iter count: 1.0 +[Apr 30, 2026 - 23:12:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 1.9307 iter count: 1.0 +[Apr 30, 2026 - 23:12:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 47.5273 iter count: 1.0 +[Apr 30, 2026 - 23:12:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.2587 iter count: 1.0 +[Apr 30, 2026 - 23:12:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 676.7988iter count: 1.0 +[Apr 30, 2026 - 23:12:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0065 iter count: 1.0 +[Apr 30, 2026 - 23:12:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:12:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 30, 2026 - 23:13:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.627965 37.778706 37.502071 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.794922 64.871094 64.769531 +util 96.750000 99.000000 92.000000 +[Apr 30, 2026 - 23:13:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 51 : avg iteration time 48.04 seconds +[Apr 30, 2026 - 23:13:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 51 : data loading time 0.00 +[Apr 30, 2026 - 23:13:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 51 : avg forward pass time 13.09 +[Apr 30, 2026 - 23:13:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 51 : backward pass time 10.13 +[Apr 30, 2026 - 23:13:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 51 : optimizer step time 1.58 +[Apr 30, 2026 - 23:13:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 51-------------------- +[Apr 30, 2026 - 23:13:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0070 iter count: 1.0 +[Apr 30, 2026 - 23:13:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0070 iter count: 1.0 +[Apr 30, 2026 - 23:13:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:13:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 23:14:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.627991 37.778709 37.502071 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.775391 64.792969 64.769531 +util 95.500000 100.000000 88.000000 +[Apr 30, 2026 - 23:14:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 52 : avg iteration time 46.44 seconds +[Apr 30, 2026 - 23:14:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 52 : data loading time 0.00 +[Apr 30, 2026 - 23:14:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 52 : avg forward pass time 12.29 +[Apr 30, 2026 - 23:14:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 52 : backward pass time 10.13 +[Apr 30, 2026 - 23:14:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 52 : optimizer step time 1.58 +[Apr 30, 2026 - 23:14:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 52-------------------- +[Apr 30, 2026 - 23:14:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0078 iter count: 1.0 +[Apr 30, 2026 - 23:14:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0078 iter count: 1.0 +[Apr 30, 2026 - 23:14:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:14:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 23:15:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.628029 37.778721 37.502071 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.682617 64.714844 64.671875 +util 91.250000 98.000000 88.000000 +[Apr 30, 2026 - 23:15:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 53 : avg iteration time 46.44 seconds +[Apr 30, 2026 - 23:15:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 53 : data loading time 0.00 +[Apr 30, 2026 - 23:15:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 53 : avg forward pass time 12.29 +[Apr 30, 2026 - 23:15:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 53 : backward pass time 10.15 +[Apr 30, 2026 - 23:15:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 53 : optimizer step time 1.57 +[Apr 30, 2026 - 23:15:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 53-------------------- +[Apr 30, 2026 - 23:15:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0065 iter count: 1.0 +[Apr 30, 2026 - 23:15:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0065 iter count: 1.0 +[Apr 30, 2026 - 23:15:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:15:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 23:15:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.628260 37.779453 37.502071 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.604492 64.753906 64.554688 +util 97.750000 100.000000 93.000000 +[Apr 30, 2026 - 23:15:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 54 : avg iteration time 46.33 seconds +[Apr 30, 2026 - 23:15:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 54 : data loading time 0.00 +[Apr 30, 2026 - 23:15:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 54 : avg forward pass time 12.25 +[Apr 30, 2026 - 23:15:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 54 : backward pass time 10.11 +[Apr 30, 2026 - 23:15:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 54 : optimizer step time 1.57 +[Apr 30, 2026 - 23:15:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 54-------------------- +[Apr 30, 2026 - 23:15:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0069 iter count: 1.0 +[Apr 30, 2026 - 23:15:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0069 iter count: 1.0 +[Apr 30, 2026 - 23:15:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:15:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 23:18:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.628288 37.778595 37.502117 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.653809 133.669922 133.648438 +util 63.750000 68.000000 58.000000 +[Apr 30, 2026 - 23:18:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 55 : avg iteration time 166.97 seconds +[Apr 30, 2026 - 23:18:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 55 : data loading time 0.00 +[Apr 30, 2026 - 23:18:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 55 : avg forward pass time 74.86 +[Apr 30, 2026 - 23:18:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 55 : backward pass time 8.42 +[Apr 30, 2026 - 23:18:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 55 : optimizer step time 0.40 +[Apr 30, 2026 - 23:18:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 55-------------------- +[Apr 30, 2026 - 23:18:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 50714.3199 iter count: 1.0 +[Apr 30, 2026 - 23:18:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7588 iter count: 1.0 +[Apr 30, 2026 - 23:18:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 50714.3199iter count: 1.0 +[Apr 30, 2026 - 23:18:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:18:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 3.7044iter count: 1.0 +[Apr 30, 2026 - 23:18:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.9918 iter count: 1.0 +[Apr 30, 2026 - 23:18:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 6.8323 iter count: 1.0 +[Apr 30, 2026 - 23:18:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 63507.8008iter count: 1.0 +[Apr 30, 2026 - 23:18:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 7.2688 iter count: 1.0 +[Apr 30, 2026 - 23:18:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 859915.8750iter count: 1.0 +[Apr 30, 2026 - 23:18:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7588iter count: 1.0 +[Apr 30, 2026 - 23:18:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 6.8323 iter count: 1.0 +[Apr 30, 2026 - 23:18:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 859915.8750iter count: 1.0 +[Apr 30, 2026 - 23:18:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 2.1166 iter count: 1.0 +[Apr 30, 2026 - 23:18:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 68.9325 iter count: 1.0 +[Apr 30, 2026 - 23:18:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.1307 iter count: 1.0 +[Apr 30, 2026 - 23:18:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 3871.6370iter count: 1.0 +[Apr 30, 2026 - 23:18:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.0354 iter count: 1.0 +[Apr 30, 2026 - 23:18:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 432.9910iter count: 1.0 +[Apr 30, 2026 - 23:18:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0070 iter count: 1.0 +[Apr 30, 2026 - 23:18:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:18:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 30, 2026 - 23:19:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.628342 37.778828 37.502117 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.980469 65.023438 64.851562 +util 97.250000 99.000000 92.000000 +[Apr 30, 2026 - 23:19:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 56 : avg iteration time 45.43 seconds +[Apr 30, 2026 - 23:19:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 56 : data loading time 0.00 +[Apr 30, 2026 - 23:19:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 56 : avg forward pass time 11.78 +[Apr 30, 2026 - 23:19:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 56 : backward pass time 10.13 +[Apr 30, 2026 - 23:19:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 56 : optimizer step time 1.57 +[Apr 30, 2026 - 23:19:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 56-------------------- +[Apr 30, 2026 - 23:19:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0064 iter count: 1.0 +[Apr 30, 2026 - 23:19:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0064 iter count: 1.0 +[Apr 30, 2026 - 23:19:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:19:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 23:20:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.628605 37.779808 37.502140 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.951172 65.003906 64.792969 +util 94.000000 100.000000 91.000000 +[Apr 30, 2026 - 23:20:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 57 : avg iteration time 49.10 seconds +[Apr 30, 2026 - 23:20:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 57 : data loading time 0.00 +[Apr 30, 2026 - 23:20:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 57 : avg forward pass time 13.63 +[Apr 30, 2026 - 23:20:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 57 : backward pass time 10.13 +[Apr 30, 2026 - 23:20:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 57 : optimizer step time 1.58 +[Apr 30, 2026 - 23:20:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 57-------------------- +[Apr 30, 2026 - 23:20:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0077 iter count: 1.0 +[Apr 30, 2026 - 23:20:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0077 iter count: 1.0 +[Apr 30, 2026 - 23:20:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:20:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 23:21:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.628606 37.779816 37.502140 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.936523 65.003906 64.734375 +util 95.500000 100.000000 88.000000 +[Apr 30, 2026 - 23:21:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 58 : avg iteration time 46.33 seconds +[Apr 30, 2026 - 23:21:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 58 : data loading time 0.00 +[Apr 30, 2026 - 23:21:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 58 : avg forward pass time 12.24 +[Apr 30, 2026 - 23:21:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 58 : backward pass time 10.14 +[Apr 30, 2026 - 23:21:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 58 : optimizer step time 1.57 +[Apr 30, 2026 - 23:21:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 58-------------------- +[Apr 30, 2026 - 23:21:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0064 iter count: 1.0 +[Apr 30, 2026 - 23:21:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0064 iter count: 1.0 +[Apr 30, 2026 - 23:21:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:21:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 23:21:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.628633 37.779850 37.502140 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.580078 64.773438 64.515625 +util 96.250000 99.000000 90.000000 +[Apr 30, 2026 - 23:21:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 59 : avg iteration time 46.42 seconds +[Apr 30, 2026 - 23:21:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 59 : data loading time 0.00 +[Apr 30, 2026 - 23:21:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 59 : avg forward pass time 12.28 +[Apr 30, 2026 - 23:21:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 59 : backward pass time 10.15 +[Apr 30, 2026 - 23:21:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 59 : optimizer step time 1.58 +[Apr 30, 2026 - 23:21:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 59-------------------- +[Apr 30, 2026 - 23:21:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0063 iter count: 1.0 +[Apr 30, 2026 - 23:21:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0063 iter count: 1.0 +[Apr 30, 2026 - 23:21:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:21:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 23:24:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.628430 37.778839 37.502163 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.627930 133.689453 133.607422 +util 59.750000 67.000000 56.000000 +[Apr 30, 2026 - 23:24:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 60 : avg iteration time 166.62 seconds +[Apr 30, 2026 - 23:24:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 60 : data loading time 0.00 +[Apr 30, 2026 - 23:24:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 60 : avg forward pass time 74.72 +[Apr 30, 2026 - 23:24:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 60 : backward pass time 8.42 +[Apr 30, 2026 - 23:24:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 60 : optimizer step time 0.40 +[Apr 30, 2026 - 23:24:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 60-------------------- +[Apr 30, 2026 - 23:24:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 16404.9978 iter count: 1.0 +[Apr 30, 2026 - 23:24:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7432 iter count: 1.0 +[Apr 30, 2026 - 23:24:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 16404.9978iter count: 1.0 +[Apr 30, 2026 - 23:24:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:24:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 3.9114iter count: 1.0 +[Apr 30, 2026 - 23:24:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.2762 iter count: 1.0 +[Apr 30, 2026 - 23:24:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 6.1763 iter count: 1.0 +[Apr 30, 2026 - 23:24:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 23047.9375iter count: 1.0 +[Apr 30, 2026 - 23:24:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 12.8376iter count: 1.0 +[Apr 30, 2026 - 23:24:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 231547.8750iter count: 1.0 +[Apr 30, 2026 - 23:24:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7432iter count: 1.0 +[Apr 30, 2026 - 23:24:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.1015 iter count: 1.0 +[Apr 30, 2026 - 23:24:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 494.2668iter count: 1.0 +[Apr 30, 2026 - 23:24:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.1012 iter count: 1.0 +[Apr 30, 2026 - 23:24:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 26968.7383iter count: 1.0 +[Apr 30, 2026 - 23:24:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 2.7198 iter count: 1.0 +[Apr 30, 2026 - 23:24:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 230.3597iter count: 1.0 +[Apr 30, 2026 - 23:24:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.6522 iter count: 1.0 +[Apr 30, 2026 - 23:24:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 10987.0850iter count: 1.0 +[Apr 30, 2026 - 23:24:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0067 iter count: 1.0 +[Apr 30, 2026 - 23:24:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:24:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 30, 2026 - 23:25:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.628489 37.778934 37.502277 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.814453 64.906250 64.539062 +util 96.000000 100.000000 93.000000 +[Apr 30, 2026 - 23:25:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 61 : avg iteration time 50.87 seconds +[Apr 30, 2026 - 23:25:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 61 : data loading time 0.00 +[Apr 30, 2026 - 23:25:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 61 : avg forward pass time 14.52 +[Apr 30, 2026 - 23:25:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 61 : backward pass time 10.12 +[Apr 30, 2026 - 23:25:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 61 : optimizer step time 1.57 +[Apr 30, 2026 - 23:25:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 61-------------------- +[Apr 30, 2026 - 23:25:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[Apr 30, 2026 - 23:25:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[Apr 30, 2026 - 23:25:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:25:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 23:26:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.628682 37.779701 37.502281 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.838867 64.988281 64.789062 +util 93.000000 99.000000 90.000000 +[Apr 30, 2026 - 23:26:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 62 : avg iteration time 46.36 seconds +[Apr 30, 2026 - 23:26:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 62 : data loading time 0.00 +[Apr 30, 2026 - 23:26:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 62 : avg forward pass time 12.24 +[Apr 30, 2026 - 23:26:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 62 : backward pass time 10.13 +[Apr 30, 2026 - 23:26:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 62 : optimizer step time 1.58 +[Apr 30, 2026 - 23:26:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 62-------------------- +[Apr 30, 2026 - 23:26:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[Apr 30, 2026 - 23:26:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[Apr 30, 2026 - 23:26:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:26:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 23:26:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.628694 37.779709 37.502335 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.755859 64.828125 64.539062 +util 93.250000 99.000000 90.000000 +[Apr 30, 2026 - 23:26:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 63 : avg iteration time 43.60 seconds +[Apr 30, 2026 - 23:26:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 63 : data loading time 0.00 +[Apr 30, 2026 - 23:26:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 63 : avg forward pass time 10.87 +[Apr 30, 2026 - 23:26:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 63 : backward pass time 10.13 +[Apr 30, 2026 - 23:26:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 63 : optimizer step time 1.57 +[Apr 30, 2026 - 23:26:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 63-------------------- +[Apr 30, 2026 - 23:26:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0069 iter count: 1.0 +[Apr 30, 2026 - 23:26:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0069 iter count: 1.0 +[Apr 30, 2026 - 23:26:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:26:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 23:27:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.628692 37.779713 37.502331 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.790039 64.867188 64.558594 +util 97.500000 100.000000 92.000000 +[Apr 30, 2026 - 23:27:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 64 : avg iteration time 46.31 seconds +[Apr 30, 2026 - 23:27:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 64 : data loading time 0.00 +[Apr 30, 2026 - 23:27:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 64 : avg forward pass time 12.21 +[Apr 30, 2026 - 23:27:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 64 : backward pass time 10.15 +[Apr 30, 2026 - 23:27:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 64 : optimizer step time 1.59 +[Apr 30, 2026 - 23:27:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 64-------------------- +[Apr 30, 2026 - 23:27:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0082 iter count: 1.0 +[Apr 30, 2026 - 23:27:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[Apr 30, 2026 - 23:27:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:27:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 23:30:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.628654 37.779449 37.502335 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.627930 133.630859 133.626953 +util 63.000000 70.000000 59.000000 +[Apr 30, 2026 - 23:30:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 65 : avg iteration time 169.70 seconds +[Apr 30, 2026 - 23:30:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 65 : data loading time 0.00 +[Apr 30, 2026 - 23:30:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 65 : avg forward pass time 76.21 +[Apr 30, 2026 - 23:30:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 65 : backward pass time 8.49 +[Apr 30, 2026 - 23:30:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 65 : optimizer step time 0.39 +[Apr 30, 2026 - 23:30:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 65-------------------- +[Apr 30, 2026 - 23:30:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 287877.5195 iter count: 1.0 +[Apr 30, 2026 - 23:30:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7300 iter count: 1.0 +[Apr 30, 2026 - 23:30:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 287877.5195iter count: 1.0 +[Apr 30, 2026 - 23:30:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:30:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.4831iter count: 1.0 +[Apr 30, 2026 - 23:30:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.6212 iter count: 1.0 +[Apr 30, 2026 - 23:30:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.6901 iter count: 1.0 +[Apr 30, 2026 - 23:30:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 373429.5938iter count: 1.0 +[Apr 30, 2026 - 23:30:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 25.5970iter count: 1.0 +[Apr 30, 2026 - 23:30:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 4781160.0000iter count: 1.0 +[Apr 30, 2026 - 23:30:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7300iter count: 1.0 +[Apr 30, 2026 - 23:30:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.6434 iter count: 1.0 +[Apr 30, 2026 - 23:30:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 1460.7441iter count: 1.0 +[Apr 30, 2026 - 23:30:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.0141 iter count: 1.0 +[Apr 30, 2026 - 23:30:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 3066.0950iter count: 1.0 +[Apr 30, 2026 - 23:30:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.9230 iter count: 1.0 +[Apr 30, 2026 - 23:30:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 2555.5957iter count: 1.0 +[Apr 30, 2026 - 23:30:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.6590 iter count: 1.0 +[Apr 30, 2026 - 23:30:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 1507.2885iter count: 1.0 +[Apr 30, 2026 - 23:30:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0077 iter count: 1.0 +[Apr 30, 2026 - 23:30:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:30:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 30, 2026 - 23:31:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.628669 37.779430 37.502338 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.843750 64.890625 64.828125 +util 91.250000 98.000000 88.000000 +[Apr 30, 2026 - 23:31:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 66 : avg iteration time 47.97 seconds +[Apr 30, 2026 - 23:31:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 66 : data loading time 0.00 +[Apr 30, 2026 - 23:31:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 66 : avg forward pass time 13.03 +[Apr 30, 2026 - 23:31:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 66 : backward pass time 10.17 +[Apr 30, 2026 - 23:31:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 66 : optimizer step time 1.57 +[Apr 30, 2026 - 23:31:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 66-------------------- +[Apr 30, 2026 - 23:31:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0080 iter count: 1.0 +[Apr 30, 2026 - 23:31:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0080 iter count: 1.0 +[Apr 30, 2026 - 23:31:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:31:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 23:32:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.628922 37.780411 37.502354 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.760742 64.851562 64.730469 +util 93.000000 99.000000 90.000000 +[Apr 30, 2026 - 23:32:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 67 : avg iteration time 46.42 seconds +[Apr 30, 2026 - 23:32:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 67 : data loading time 0.00 +[Apr 30, 2026 - 23:32:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 67 : avg forward pass time 12.26 +[Apr 30, 2026 - 23:32:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 67 : backward pass time 10.15 +[Apr 30, 2026 - 23:32:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 67 : optimizer step time 1.57 +[Apr 30, 2026 - 23:32:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 67-------------------- +[Apr 30, 2026 - 23:32:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0081 iter count: 1.0 +[Apr 30, 2026 - 23:32:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[Apr 30, 2026 - 23:32:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:32:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 23:32:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.629041 37.780659 37.502487 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.814453 64.828125 64.773438 +util 93.250000 97.000000 88.000000 +[Apr 30, 2026 - 23:32:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 68 : avg iteration time 49.06 seconds +[Apr 30, 2026 - 23:32:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 68 : data loading time 0.00 +[Apr 30, 2026 - 23:32:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 68 : avg forward pass time 13.60 +[Apr 30, 2026 - 23:32:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 68 : backward pass time 10.13 +[Apr 30, 2026 - 23:32:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 68 : optimizer step time 1.58 +[Apr 30, 2026 - 23:32:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 68-------------------- +[Apr 30, 2026 - 23:32:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0079 iter count: 1.0 +[Apr 30, 2026 - 23:32:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0079 iter count: 1.0 +[Apr 30, 2026 - 23:32:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:32:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 23:33:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.628911 37.780109 37.502502 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.721680 64.753906 64.710938 +util 95.500000 100.000000 88.000000 +[Apr 30, 2026 - 23:33:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 69 : avg iteration time 49.09 seconds +[Apr 30, 2026 - 23:33:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 69 : data loading time 0.00 +[Apr 30, 2026 - 23:33:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 69 : avg forward pass time 13.60 +[Apr 30, 2026 - 23:33:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 69 : backward pass time 10.15 +[Apr 30, 2026 - 23:33:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 69 : optimizer step time 1.57 +[Apr 30, 2026 - 23:33:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 69-------------------- +[Apr 30, 2026 - 23:33:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0073 iter count: 1.0 +[Apr 30, 2026 - 23:33:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0073 iter count: 1.0 +[Apr 30, 2026 - 23:33:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:33:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 23:36:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.628717 37.779057 37.502670 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.583984 133.587891 133.572266 +util 64.750000 72.000000 61.000000 +[Apr 30, 2026 - 23:36:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 70 : avg iteration time 166.91 seconds +[Apr 30, 2026 - 23:36:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 70 : data loading time 0.00 +[Apr 30, 2026 - 23:36:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 70 : avg forward pass time 74.84 +[Apr 30, 2026 - 23:36:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 70 : backward pass time 8.42 +[Apr 30, 2026 - 23:36:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 70 : optimizer step time 0.39 +[Apr 30, 2026 - 23:36:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 70-------------------- +[Apr 30, 2026 - 23:36:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 200599.8066 iter count: 1.0 +[Apr 30, 2026 - 23:36:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7362 iter count: 1.0 +[Apr 30, 2026 - 23:36:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 200599.8066iter count: 1.0 +[Apr 30, 2026 - 23:36:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:36:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.3596iter count: 1.0 +[Apr 30, 2026 - 23:36:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.0594 iter count: 1.0 +[Apr 30, 2026 - 23:36:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.8895 iter count: 1.0 +[Apr 30, 2026 - 23:36:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 311931.0000iter count: 1.0 +[Apr 30, 2026 - 23:36:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 8.3213 iter count: 1.0 +[Apr 30, 2026 - 23:36:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 7124090.5000iter count: 1.0 +[Apr 30, 2026 - 23:36:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7362iter count: 1.0 +[Apr 30, 2026 - 23:36:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 2.5176 iter count: 1.0 +[Apr 30, 2026 - 23:36:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 153.7441iter count: 1.0 +[Apr 30, 2026 - 23:36:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.4651 iter count: 1.0 +[Apr 30, 2026 - 23:36:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 7556.2188iter count: 1.0 +[Apr 30, 2026 - 23:36:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 1.8220 iter count: 1.0 +[Apr 30, 2026 - 23:36:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 38.2448 iter count: 1.0 +[Apr 30, 2026 - 23:36:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.2051 iter count: 1.0 +[Apr 30, 2026 - 23:36:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 4492.3340iter count: 1.0 +[Apr 30, 2026 - 23:36:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0066 iter count: 1.0 +[Apr 30, 2026 - 23:36:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:36:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 30, 2026 - 23:37:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.628929 37.779785 37.502716 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.633789 64.929688 64.535156 +util 96.750000 99.000000 92.000000 +[Apr 30, 2026 - 23:37:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 71 : avg iteration time 48.07 seconds +[Apr 30, 2026 - 23:37:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 71 : data loading time 0.00 +[Apr 30, 2026 - 23:37:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 71 : avg forward pass time 13.10 +[Apr 30, 2026 - 23:37:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 71 : backward pass time 10.13 +[Apr 30, 2026 - 23:37:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 71 : optimizer step time 1.59 +[Apr 30, 2026 - 23:37:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 71-------------------- +[Apr 30, 2026 - 23:37:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0075 iter count: 1.0 +[Apr 30, 2026 - 23:37:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0075 iter count: 1.0 +[Apr 30, 2026 - 23:37:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:37:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 23:38:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.629019 37.780052 37.502739 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.604492 64.929688 64.496094 +util 96.000000 100.000000 90.000000 +[Apr 30, 2026 - 23:38:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 72 : avg iteration time 46.39 seconds +[Apr 30, 2026 - 23:38:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 72 : data loading time 0.00 +[Apr 30, 2026 - 23:38:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 72 : avg forward pass time 12.25 +[Apr 30, 2026 - 23:38:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 72 : backward pass time 10.15 +[Apr 30, 2026 - 23:38:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 72 : optimizer step time 1.58 +[Apr 30, 2026 - 23:38:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 72-------------------- +[Apr 30, 2026 - 23:38:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0076 iter count: 1.0 +[Apr 30, 2026 - 23:38:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0076 iter count: 1.0 +[Apr 30, 2026 - 23:38:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:38:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 23:38:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.629070 37.780304 37.502693 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.604492 64.929688 64.496094 +util 98.000000 100.000000 94.000000 +[Apr 30, 2026 - 23:38:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 73 : avg iteration time 49.04 seconds +[Apr 30, 2026 - 23:38:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 73 : data loading time 0.00 +[Apr 30, 2026 - 23:38:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 73 : avg forward pass time 13.58 +[Apr 30, 2026 - 23:38:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 73 : backward pass time 10.14 +[Apr 30, 2026 - 23:38:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 73 : optimizer step time 1.58 +[Apr 30, 2026 - 23:38:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 73-------------------- +[Apr 30, 2026 - 23:38:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0071 iter count: 1.0 +[Apr 30, 2026 - 23:38:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0071 iter count: 1.0 +[Apr 30, 2026 - 23:38:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:38:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 23:39:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.629029 37.780014 37.502739 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.892578 64.906250 64.851562 +util 97.750000 100.000000 93.000000 +[Apr 30, 2026 - 23:39:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 74 : avg iteration time 49.19 seconds +[Apr 30, 2026 - 23:39:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 74 : data loading time 0.00 +[Apr 30, 2026 - 23:39:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 74 : avg forward pass time 13.65 +[Apr 30, 2026 - 23:39:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 74 : backward pass time 10.15 +[Apr 30, 2026 - 23:39:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 74 : optimizer step time 1.58 +[Apr 30, 2026 - 23:39:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 74-------------------- +[Apr 30, 2026 - 23:39:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0077 iter count: 1.0 +[Apr 30, 2026 - 23:39:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0077 iter count: 1.0 +[Apr 30, 2026 - 23:39:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:39:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 23:42:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.629229 37.779072 37.503239 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.618652 133.626953 133.593750 +util 63.500000 71.000000 60.000000 +[Apr 30, 2026 - 23:42:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 75 : avg iteration time 166.91 seconds +[Apr 30, 2026 - 23:42:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 75 : data loading time 0.00 +[Apr 30, 2026 - 23:42:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 75 : avg forward pass time 74.79 +[Apr 30, 2026 - 23:42:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 75 : backward pass time 8.48 +[Apr 30, 2026 - 23:42:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 75 : optimizer step time 0.40 +[Apr 30, 2026 - 23:42:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 75-------------------- +[Apr 30, 2026 - 23:42:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 92176.4360 iter count: 1.0 +[Apr 30, 2026 - 23:42:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6764 iter count: 1.0 +[Apr 30, 2026 - 23:42:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 92176.4360iter count: 1.0 +[Apr 30, 2026 - 23:42:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:42:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.4737iter count: 1.0 +[Apr 30, 2026 - 23:42:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.6374 iter count: 1.0 +[Apr 30, 2026 - 23:42:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.1999 iter count: 1.0 +[Apr 30, 2026 - 23:42:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 127054.0781iter count: 1.0 +[Apr 30, 2026 - 23:42:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 3.5782 iter count: 1.0 +[Apr 30, 2026 - 23:42:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 1793892.7500iter count: 1.0 +[Apr 30, 2026 - 23:42:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6764iter count: 1.0 +[Apr 30, 2026 - 23:42:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.4934 iter count: 1.0 +[Apr 30, 2026 - 23:42:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 7996.0728iter count: 1.0 +[Apr 30, 2026 - 23:42:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.3920 iter count: 1.0 +[Apr 30, 2026 - 23:42:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 883.5829iter count: 1.0 +[Apr 30, 2026 - 23:42:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 2.1985 iter count: 1.0 +[Apr 30, 2026 - 23:42:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 81.2020 iter count: 1.0 +[Apr 30, 2026 - 23:42:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.7373 iter count: 1.0 +[Apr 30, 2026 - 23:42:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 13024.2637iter count: 1.0 +[Apr 30, 2026 - 23:42:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0069 iter count: 1.0 +[Apr 30, 2026 - 23:42:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:42:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 30, 2026 - 23:43:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.629354 37.779579 37.503231 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.873047 64.964844 64.597656 +util 95.250000 99.000000 88.000000 +[Apr 30, 2026 - 23:43:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 76 : avg iteration time 50.73 seconds +[Apr 30, 2026 - 23:43:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 76 : data loading time 0.00 +[Apr 30, 2026 - 23:43:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 76 : avg forward pass time 14.42 +[Apr 30, 2026 - 23:43:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 76 : backward pass time 10.14 +[Apr 30, 2026 - 23:43:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 76 : optimizer step time 1.59 +[Apr 30, 2026 - 23:43:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 76-------------------- +[Apr 30, 2026 - 23:43:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0082 iter count: 1.0 +[Apr 30, 2026 - 23:43:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[Apr 30, 2026 - 23:43:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:43:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 23:44:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.629550 37.780342 37.503235 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.956055 65.046875 64.925781 +util 92.000000 99.000000 88.000000 +[Apr 30, 2026 - 23:44:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 77 : avg iteration time 49.24 seconds +[Apr 30, 2026 - 23:44:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 77 : data loading time 0.00 +[Apr 30, 2026 - 23:44:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 77 : avg forward pass time 13.68 +[Apr 30, 2026 - 23:44:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 77 : backward pass time 10.14 +[Apr 30, 2026 - 23:44:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 77 : optimizer step time 1.58 +[Apr 30, 2026 - 23:44:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 77-------------------- +[Apr 30, 2026 - 23:44:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[Apr 30, 2026 - 23:44:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[Apr 30, 2026 - 23:44:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:44:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 23:45:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.629613 37.780586 37.503216 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.829102 64.906250 64.597656 +util 93.000000 99.000000 86.000000 +[Apr 30, 2026 - 23:45:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 78 : avg iteration time 49.18 seconds +[Apr 30, 2026 - 23:45:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 78 : data loading time 0.00 +[Apr 30, 2026 - 23:45:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 78 : avg forward pass time 13.65 +[Apr 30, 2026 - 23:45:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 78 : backward pass time 10.15 +[Apr 30, 2026 - 23:45:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 78 : optimizer step time 1.58 +[Apr 30, 2026 - 23:45:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 78-------------------- +[Apr 30, 2026 - 23:45:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0079 iter count: 1.0 +[Apr 30, 2026 - 23:45:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0079 iter count: 1.0 +[Apr 30, 2026 - 23:45:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:45:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 23:45:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.629553 37.780312 37.503216 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.946289 65.066406 64.906250 +util 96.000000 100.000000 93.000000 +[Apr 30, 2026 - 23:45:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 79 : avg iteration time 49.08 seconds +[Apr 30, 2026 - 23:45:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 79 : data loading time 0.00 +[Apr 30, 2026 - 23:45:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 79 : avg forward pass time 13.61 +[Apr 30, 2026 - 23:45:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 79 : backward pass time 10.14 +[Apr 30, 2026 - 23:45:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 79 : optimizer step time 1.58 +[Apr 30, 2026 - 23:45:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 79-------------------- +[Apr 30, 2026 - 23:45:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[Apr 30, 2026 - 23:45:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[Apr 30, 2026 - 23:45:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:45:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 23:48:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.630078 37.780987 37.503719 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.590820 133.609375 133.535156 +util 63.500000 72.000000 56.000000 +[Apr 30, 2026 - 23:48:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 80 : avg iteration time 161.33 seconds +[Apr 30, 2026 - 23:48:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 80 : data loading time 0.00 +[Apr 30, 2026 - 23:48:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 80 : avg forward pass time 72.04 +[Apr 30, 2026 - 23:48:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 80 : backward pass time 8.46 +[Apr 30, 2026 - 23:48:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 80 : optimizer step time 0.39 +[Apr 30, 2026 - 23:48:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 80-------------------- +[Apr 30, 2026 - 23:48:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 230686.7266 iter count: 1.0 +[Apr 30, 2026 - 23:48:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7228 iter count: 1.0 +[Apr 30, 2026 - 23:48:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 230686.7266iter count: 1.0 +[Apr 30, 2026 - 23:48:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:48:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 3.7318iter count: 1.0 +[Apr 30, 2026 - 23:48:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.3882 iter count: 1.0 +[Apr 30, 2026 - 23:48:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.0084 iter count: 1.0 +[Apr 30, 2026 - 23:48:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 346673.9375iter count: 1.0 +[Apr 30, 2026 - 23:48:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 2.1738 iter count: 1.0 +[Apr 30, 2026 - 23:48:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 9037244.0000iter count: 1.0 +[Apr 30, 2026 - 23:48:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7228iter count: 1.0 +[Apr 30, 2026 - 23:48:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 0.3882 iter count: 1.0 +[Apr 30, 2026 - 23:48:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 2.1738 iter count: 1.0 +[Apr 30, 2026 - 23:48:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 6.7549 iter count: 1.0 +[Apr 30, 2026 - 23:48:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 736560.4375iter count: 1.0 +[Apr 30, 2026 - 23:48:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 6.0630 iter count: 1.0 +[Apr 30, 2026 - 23:48:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 184604.6562iter count: 1.0 +[Apr 30, 2026 - 23:48:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 2.5336 iter count: 1.0 +[Apr 30, 2026 - 23:48:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 158.7389iter count: 1.0 +[Apr 30, 2026 - 23:48:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0072 iter count: 1.0 +[Apr 30, 2026 - 23:48:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:48:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 30, 2026 - 23:49:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.630087 37.780987 37.503716 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.609375 64.714844 64.574219 +util 95.250000 100.000000 92.000000 +[Apr 30, 2026 - 23:49:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 81 : avg iteration time 45.26 seconds +[Apr 30, 2026 - 23:49:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 81 : data loading time 0.00 +[Apr 30, 2026 - 23:49:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 81 : avg forward pass time 11.70 +[Apr 30, 2026 - 23:49:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 81 : backward pass time 10.13 +[Apr 30, 2026 - 23:49:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 81 : optimizer step time 1.58 +[Apr 30, 2026 - 23:49:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 81-------------------- +[Apr 30, 2026 - 23:49:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0079 iter count: 1.0 +[Apr 30, 2026 - 23:49:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0079 iter count: 1.0 +[Apr 30, 2026 - 23:49:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:49:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 23:50:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.630339 37.781982 37.503700 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.619141 64.753906 64.574219 +util 97.500000 99.000000 93.000000 +[Apr 30, 2026 - 23:50:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 82 : avg iteration time 43.68 seconds +[Apr 30, 2026 - 23:50:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 82 : data loading time 0.00 +[Apr 30, 2026 - 23:50:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 82 : avg forward pass time 10.92 +[Apr 30, 2026 - 23:50:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 82 : backward pass time 10.14 +[Apr 30, 2026 - 23:50:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 82 : optimizer step time 1.57 +[Apr 30, 2026 - 23:50:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 82-------------------- +[Apr 30, 2026 - 23:50:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0085 iter count: 1.0 +[Apr 30, 2026 - 23:50:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[Apr 30, 2026 - 23:50:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:50:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 23:50:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.630420 37.782276 37.503727 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.687500 65.203125 64.515625 +util 92.750000 99.000000 89.000000 +[Apr 30, 2026 - 23:50:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 83 : avg iteration time 46.49 seconds +[Apr 30, 2026 - 23:50:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 83 : data loading time 0.00 +[Apr 30, 2026 - 23:50:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 83 : avg forward pass time 12.32 +[Apr 30, 2026 - 23:50:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 83 : backward pass time 10.15 +[Apr 30, 2026 - 23:50:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 83 : optimizer step time 1.57 +[Apr 30, 2026 - 23:50:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 83-------------------- +[Apr 30, 2026 - 23:50:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[Apr 30, 2026 - 23:50:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[Apr 30, 2026 - 23:50:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:50:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 23:51:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.630212 37.781406 37.503727 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.677734 65.046875 64.554688 +util 95.500000 99.000000 90.000000 +[Apr 30, 2026 - 23:51:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 84 : avg iteration time 46.48 seconds +[Apr 30, 2026 - 23:51:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 84 : data loading time 0.00 +[Apr 30, 2026 - 23:51:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 84 : avg forward pass time 12.30 +[Apr 30, 2026 - 23:51:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 84 : backward pass time 10.16 +[Apr 30, 2026 - 23:51:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 84 : optimizer step time 1.57 +[Apr 30, 2026 - 23:51:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 84-------------------- +[Apr 30, 2026 - 23:51:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0084 iter count: 1.0 +[Apr 30, 2026 - 23:51:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[Apr 30, 2026 - 23:51:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:51:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 23:54:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.630036 37.780720 37.503750 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.623047 133.646484 133.552734 +util 62.750000 70.000000 59.000000 +[Apr 30, 2026 - 23:54:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 85 : avg iteration time 167.34 seconds +[Apr 30, 2026 - 23:54:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 85 : data loading time 0.00 +[Apr 30, 2026 - 23:54:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 85 : avg forward pass time 75.06 +[Apr 30, 2026 - 23:54:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 85 : backward pass time 8.43 +[Apr 30, 2026 - 23:54:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 85 : optimizer step time 0.40 +[Apr 30, 2026 - 23:54:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 85-------------------- +[Apr 30, 2026 - 23:54:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 93405.2218 iter count: 1.0 +[Apr 30, 2026 - 23:54:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7257 iter count: 1.0 +[Apr 30, 2026 - 23:54:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 93405.2218iter count: 1.0 +[Apr 30, 2026 - 23:54:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:54:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.0648iter count: 1.0 +[Apr 30, 2026 - 23:54:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.4647 iter count: 1.0 +[Apr 30, 2026 - 23:54:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.6132 iter count: 1.0 +[Apr 30, 2026 - 23:54:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 164288.8281iter count: 1.0 +[Apr 30, 2026 - 23:54:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 18.7155iter count: 1.0 +[Apr 30, 2026 - 23:54:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 4099320.2500iter count: 1.0 +[Apr 30, 2026 - 23:54:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7257iter count: 1.0 +[Apr 30, 2026 - 23:54:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.4319 iter count: 1.0 +[Apr 30, 2026 - 23:54:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 52245.2734iter count: 1.0 +[Apr 30, 2026 - 23:54:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 2.7136 iter count: 1.0 +[Apr 30, 2026 - 23:54:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 227.4953iter count: 1.0 +[Apr 30, 2026 - 23:54:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 1.7178 iter count: 1.0 +[Apr 30, 2026 - 23:54:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 31.0512 iter count: 1.0 +[Apr 30, 2026 - 23:54:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.9095 iter count: 1.0 +[Apr 30, 2026 - 23:54:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 18380.1211iter count: 1.0 +[Apr 30, 2026 - 23:54:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0080 iter count: 1.0 +[Apr 30, 2026 - 23:54:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:54:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 30, 2026 - 23:55:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.630065 37.780743 37.503750 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.638672 64.949219 64.535156 +util 93.000000 99.000000 90.000000 +[Apr 30, 2026 - 23:55:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 86 : avg iteration time 47.98 seconds +[Apr 30, 2026 - 23:55:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 86 : data loading time 0.00 +[Apr 30, 2026 - 23:55:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 86 : avg forward pass time 13.08 +[Apr 30, 2026 - 23:55:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 86 : backward pass time 10.12 +[Apr 30, 2026 - 23:55:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 86 : optimizer step time 1.57 +[Apr 30, 2026 - 23:55:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 86-------------------- +[Apr 30, 2026 - 23:55:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0069 iter count: 1.0 +[Apr 30, 2026 - 23:55:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0069 iter count: 1.0 +[Apr 30, 2026 - 23:55:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:55:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 23:55:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.630392 37.782005 37.503750 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.829102 64.890625 64.808594 +util 95.000000 99.000000 90.000000 +[Apr 30, 2026 - 23:55:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 87 : avg iteration time 46.27 seconds +[Apr 30, 2026 - 23:55:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 87 : data loading time 0.00 +[Apr 30, 2026 - 23:55:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 87 : avg forward pass time 12.21 +[Apr 30, 2026 - 23:55:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 87 : backward pass time 10.15 +[Apr 30, 2026 - 23:55:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 87 : optimizer step time 1.57 +[Apr 30, 2026 - 23:55:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 87-------------------- +[Apr 30, 2026 - 23:55:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0078 iter count: 1.0 +[Apr 30, 2026 - 23:55:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0078 iter count: 1.0 +[Apr 30, 2026 - 23:55:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:55:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 23:56:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.630391 37.782005 37.503750 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.799805 64.949219 64.750000 +util 93.250000 100.000000 90.000000 +[Apr 30, 2026 - 23:56:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 88 : avg iteration time 46.15 seconds +[Apr 30, 2026 - 23:56:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 88 : data loading time 0.00 +[Apr 30, 2026 - 23:56:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 88 : avg forward pass time 12.17 +[Apr 30, 2026 - 23:56:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 88 : backward pass time 10.12 +[Apr 30, 2026 - 23:56:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 88 : optimizer step time 1.58 +[Apr 30, 2026 - 23:56:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 88-------------------- +[Apr 30, 2026 - 23:56:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0072 iter count: 1.0 +[Apr 30, 2026 - 23:56:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0072 iter count: 1.0 +[Apr 30, 2026 - 23:56:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:56:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 30, 2026 - 23:57:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.630392 37.782028 37.503750 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.770508 64.949219 64.710938 +util 96.750000 99.000000 92.000000 +[Apr 30, 2026 - 23:57:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 89 : avg iteration time 46.28 seconds +[Apr 30, 2026 - 23:57:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 89 : data loading time 0.00 +[Apr 30, 2026 - 23:57:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 89 : avg forward pass time 12.22 +[Apr 30, 2026 - 23:57:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 89 : backward pass time 10.12 +[Apr 30, 2026 - 23:57:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 89 : optimizer step time 1.58 +[Apr 30, 2026 - 23:57:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 89-------------------- +[Apr 30, 2026 - 23:57:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0072 iter count: 1.0 +[Apr 30, 2026 - 23:57:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0072 iter count: 1.0 +[Apr 30, 2026 - 23:57:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 30, 2026 - 23:57:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 00:00:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.629918 37.780109 37.503757 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.574219 133.591797 133.568359 +util 64.750000 69.000000 60.000000 +[May 1, 2026 - 00:00:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 90 : avg iteration time 169.40 seconds +[May 1, 2026 - 00:00:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 90 : data loading time 0.00 +[May 1, 2026 - 00:00:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 90 : avg forward pass time 76.11 +[May 1, 2026 - 00:00:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 90 : backward pass time 8.41 +[May 1, 2026 - 00:00:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 90 : optimizer step time 0.40 +[May 1, 2026 - 00:00:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 90-------------------- +[May 1, 2026 - 00:00:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 115330.6830 iter count: 1.0 +[May 1, 2026 - 00:00:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7252 iter count: 1.0 +[May 1, 2026 - 00:00:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 115330.6830iter count: 1.0 +[May 1, 2026 - 00:00:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:00:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.2108iter count: 1.0 +[May 1, 2026 - 00:00:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.8451 iter count: 1.0 +[May 1, 2026 - 00:00:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.5576 iter count: 1.0 +[May 1, 2026 - 00:00:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 186810.0312iter count: 1.0 +[May 1, 2026 - 00:00:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 5.4206 iter count: 1.0 +[May 1, 2026 - 00:00:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 3668259.0000iter count: 1.0 +[May 1, 2026 - 00:00:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7252iter count: 1.0 +[May 1, 2026 - 00:00:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 2.5001 iter count: 1.0 +[May 1, 2026 - 00:00:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 148.4409iter count: 1.0 +[May 1, 2026 - 00:00:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 6.1473 iter count: 1.0 +[May 1, 2026 - 00:00:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 218517.9375iter count: 1.0 +[May 1, 2026 - 00:00:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.2275 iter count: 1.0 +[May 1, 2026 - 00:00:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 635.8931iter count: 1.0 +[May 1, 2026 - 00:00:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.0472 iter count: 1.0 +[May 1, 2026 - 00:00:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 24204.7012iter count: 1.0 +[May 1, 2026 - 00:00:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0076 iter count: 1.0 +[May 1, 2026 - 00:00:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:00:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 00:01:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.629980 37.780357 37.503757 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.624023 64.890625 64.535156 +util 94.750000 100.000000 92.000000 +[May 1, 2026 - 00:01:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 91 : avg iteration time 45.34 seconds +[May 1, 2026 - 00:01:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 91 : data loading time 0.00 +[May 1, 2026 - 00:01:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 91 : avg forward pass time 11.74 +[May 1, 2026 - 00:01:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 91 : backward pass time 10.15 +[May 1, 2026 - 00:01:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 91 : optimizer step time 1.58 +[May 1, 2026 - 00:01:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 91-------------------- +[May 1, 2026 - 00:01:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0081 iter count: 1.0 +[May 1, 2026 - 00:01:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[May 1, 2026 - 00:01:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:01:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 00:01:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.630233 37.781368 37.503757 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.833984 64.847656 64.792969 +util 93.750000 100.000000 87.000000 +[May 1, 2026 - 00:01:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 92 : avg iteration time 49.13 seconds +[May 1, 2026 - 00:01:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 92 : data loading time 0.00 +[May 1, 2026 - 00:01:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 92 : avg forward pass time 13.64 +[May 1, 2026 - 00:01:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 92 : backward pass time 10.13 +[May 1, 2026 - 00:01:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 92 : optimizer step time 1.58 +[May 1, 2026 - 00:01:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 92-------------------- +[May 1, 2026 - 00:01:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0075 iter count: 1.0 +[May 1, 2026 - 00:01:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0075 iter count: 1.0 +[May 1, 2026 - 00:01:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:01:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 00:02:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.630237 37.781357 37.503757 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.638672 64.773438 64.593750 +util 95.250000 100.000000 92.000000 +[May 1, 2026 - 00:02:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 93 : avg iteration time 46.27 seconds +[May 1, 2026 - 00:02:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 93 : data loading time 0.00 +[May 1, 2026 - 00:02:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 93 : avg forward pass time 12.20 +[May 1, 2026 - 00:02:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 93 : backward pass time 10.14 +[May 1, 2026 - 00:02:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 93 : optimizer step time 1.58 +[May 1, 2026 - 00:02:40 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 93-------------------- +[May 1, 2026 - 00:02:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0070 iter count: 1.0 +[May 1, 2026 - 00:02:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0070 iter count: 1.0 +[May 1, 2026 - 00:02:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:02:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 00:03:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.630247 37.781361 37.503796 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.809570 64.828125 64.753906 +util 92.000000 99.000000 88.000000 +[May 1, 2026 - 00:03:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 94 : avg iteration time 43.75 seconds +[May 1, 2026 - 00:03:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 94 : data loading time 0.00 +[May 1, 2026 - 00:03:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 94 : avg forward pass time 10.96 +[May 1, 2026 - 00:03:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 94 : backward pass time 10.15 +[May 1, 2026 - 00:03:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 94 : optimizer step time 1.56 +[May 1, 2026 - 00:03:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 94-------------------- +[May 1, 2026 - 00:03:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0072 iter count: 1.0 +[May 1, 2026 - 00:03:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0072 iter count: 1.0 +[May 1, 2026 - 00:03:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:03:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 00:06:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.629847 37.779797 37.503761 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.609863 133.669922 133.589844 +util 61.500000 69.000000 57.000000 +[May 1, 2026 - 00:06:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 95 : avg iteration time 163.41 seconds +[May 1, 2026 - 00:06:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 95 : data loading time 0.00 +[May 1, 2026 - 00:06:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 95 : avg forward pass time 73.08 +[May 1, 2026 - 00:06:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 95 : backward pass time 8.45 +[May 1, 2026 - 00:06:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 95 : optimizer step time 0.39 +[May 1, 2026 - 00:06:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 95-------------------- +[May 1, 2026 - 00:06:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 740860.7993 iter count: 1.0 +[May 1, 2026 - 00:06:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7625 iter count: 1.0 +[May 1, 2026 - 00:06:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 740860.7993iter count: 1.0 +[May 1, 2026 - 00:06:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:06:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.2587iter count: 1.0 +[May 1, 2026 - 00:06:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.0226 iter count: 1.0 +[May 1, 2026 - 00:06:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.6015 iter count: 1.0 +[May 1, 2026 - 00:06:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 984132.2500iter count: 1.0 +[May 1, 2026 - 00:06:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 7.7306 iter count: 1.0 +[May 1, 2026 - 00:06:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 29592726.0000iter count: 1.0 +[May 1, 2026 - 00:06:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7625iter count: 1.0 +[May 1, 2026 - 00:06:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 1.7401 iter count: 1.0 +[May 1, 2026 - 00:06:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 32.4660 iter count: 1.0 +[May 1, 2026 - 00:06:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.1364 iter count: 1.0 +[May 1, 2026 - 00:06:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 529.9372iter count: 1.0 +[May 1, 2026 - 00:06:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.2440 iter count: 1.0 +[May 1, 2026 - 00:06:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 657.1756iter count: 1.0 +[May 1, 2026 - 00:06:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.1781 iter count: 1.0 +[May 1, 2026 - 00:06:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 31450.0957iter count: 1.0 +[May 1, 2026 - 00:06:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0074 iter count: 1.0 +[May 1, 2026 - 00:06:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:06:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 00:06:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.629980 37.780296 37.503811 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.926758 64.929688 64.925781 +util 97.000000 99.000000 92.000000 +[May 1, 2026 - 00:06:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 96 : avg iteration time 45.10 seconds +[May 1, 2026 - 00:06:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 96 : data loading time 0.00 +[May 1, 2026 - 00:06:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 96 : avg forward pass time 11.61 +[May 1, 2026 - 00:06:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 96 : backward pass time 10.14 +[May 1, 2026 - 00:06:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 96 : optimizer step time 1.57 +[May 1, 2026 - 00:06:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 96-------------------- +[May 1, 2026 - 00:06:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0069 iter count: 1.0 +[May 1, 2026 - 00:06:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0069 iter count: 1.0 +[May 1, 2026 - 00:06:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:06:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 00:07:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.630242 37.781296 37.503826 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.877930 64.886719 64.851562 +util 94.250000 99.000000 91.000000 +[May 1, 2026 - 00:07:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 97 : avg iteration time 46.31 seconds +[May 1, 2026 - 00:07:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 97 : data loading time 0.00 +[May 1, 2026 - 00:07:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 97 : avg forward pass time 12.22 +[May 1, 2026 - 00:07:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 97 : backward pass time 10.15 +[May 1, 2026 - 00:07:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 97 : optimizer step time 1.57 +[May 1, 2026 - 00:07:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 97-------------------- +[May 1, 2026 - 00:07:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0075 iter count: 1.0 +[May 1, 2026 - 00:07:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0075 iter count: 1.0 +[May 1, 2026 - 00:07:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:07:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 00:08:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.630254 37.781296 37.503811 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.809570 64.929688 64.769531 +util 96.500000 99.000000 92.000000 +[May 1, 2026 - 00:08:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 98 : avg iteration time 46.27 seconds +[May 1, 2026 - 00:08:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 98 : data loading time 0.00 +[May 1, 2026 - 00:08:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 98 : avg forward pass time 12.21 +[May 1, 2026 - 00:08:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 98 : backward pass time 10.14 +[May 1, 2026 - 00:08:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 98 : optimizer step time 1.57 +[May 1, 2026 - 00:08:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 98-------------------- +[May 1, 2026 - 00:08:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0077 iter count: 1.0 +[May 1, 2026 - 00:08:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0077 iter count: 1.0 +[May 1, 2026 - 00:08:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:08:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 00:09:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.630265 37.781296 37.503857 +peak_gpu_mem_gb 62.822126 62.825250 62.821084 +peak_gpu_mem_reserved_gb 64.873047 64.949219 64.847656 +util 93.000000 100.000000 89.000000 +[May 1, 2026 - 00:09:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 99 : avg iteration time 46.30 seconds +[May 1, 2026 - 00:09:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 99 : data loading time 0.00 +[May 1, 2026 - 00:09:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 99 : avg forward pass time 12.23 +[May 1, 2026 - 00:09:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 99 : backward pass time 10.14 +[May 1, 2026 - 00:09:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 99 : optimizer step time 1.56 +[May 1, 2026 - 00:09:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 99-------------------- +[May 1, 2026 - 00:09:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0075 iter count: 1.0 +[May 1, 2026 - 00:09:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0075 iter count: 1.0 +[May 1, 2026 - 00:09:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:09:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 00:12:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 37.629877 37.779762 37.503815 +peak_gpu_mem_gb 129.741254 129.744378 129.740212 +peak_gpu_mem_reserved_gb 133.633301 133.652344 133.626953 +util 66.500000 70.000000 62.000000 +[May 1, 2026 - 00:12:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 100 : avg iteration time 171.12 seconds +[May 1, 2026 - 00:12:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 100 : data loading time 0.00 +[May 1, 2026 - 00:12:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 100 : avg forward pass time 76.97 +[May 1, 2026 - 00:12:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 100 : backward pass time 8.41 +[May 1, 2026 - 00:12:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 100 : optimizer step time 0.42 +[May 1, 2026 - 00:12:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 100-------------------- +[May 1, 2026 - 00:12:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 82475.2075 iter count: 1.0 +[May 1, 2026 - 00:12:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7478 iter count: 1.0 +[May 1, 2026 - 00:12:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 82475.2075iter count: 1.0 +[May 1, 2026 - 00:12:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:12:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.1489iter count: 1.0 +[May 1, 2026 - 00:12:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.9506 iter count: 1.0 +[May 1, 2026 - 00:12:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.2495 iter count: 1.0 +[May 1, 2026 - 00:12:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 113506.2188iter count: 1.0 +[May 1, 2026 - 00:12:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 6.6946 iter count: 1.0 +[May 1, 2026 - 00:12:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 1980601.0000iter count: 1.0 +[May 1, 2026 - 00:12:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7478iter count: 1.0 +[May 1, 2026 - 00:12:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.2285 iter count: 1.0 +[May 1, 2026 - 00:12:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 4707.7461iter count: 1.0 +[May 1, 2026 - 00:12:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.4682 iter count: 1.0 +[May 1, 2026 - 00:12:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 56182.3242iter count: 1.0 +[May 1, 2026 - 00:12:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.7187 iter count: 1.0 +[May 1, 2026 - 00:12:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 12549.2012iter count: 1.0 +[May 1, 2026 - 00:12:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.1855 iter count: 1.0 +[May 1, 2026 - 00:12:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 31917.4668iter count: 1.0 +[May 1, 2026 - 00:12:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 1, 2026 - 00:12:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 1, 2026 - 00:12:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 18.26 seconds +[May 1, 2026 - 00:12:20 | INFO | fastgen.trainer:save_checkpoint:266 ] Saving checkpoint iteration 100 +[May 1, 2026 - 00:12:20 | INFO | fastgen.utils.checkpointer:save:311 ] Saving FSDP model to prefix /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000100 +[May 1, 2026 - 00:12:58 | SUCCESS | fastgen.utils.checkpointer:save:362 ] Model saved at iteration 100 +[May 1, 2026 - 00:12:58 | INFO | fastgen.trainer:validate:358 ] Validation iteration 100 +[May 1, 2026 - 00:12:59 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 00:13:01 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 2.1s +[May 1, 2026 - 00:13:04 | INFO | fastgen.callbacks.wandb:_get_syncnet_scorer:402 ] [WandbCallback] Loaded SyncCScorer from /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model +[May 1, 2026 - 00:13:04 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 0: sync_c=6.454 +[May 1, 2026 - 00:13:05 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 00:13:07 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 00:13:09 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 1: sync_c=5.885 +[May 1, 2026 - 00:13:10 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 00:13:12 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 00:13:15 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 2: sync_c=5.006 +[May 1, 2026 - 00:13:15 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 00:13:17 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 00:13:20 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 3: sync_c=4.267 +[May 1, 2026 - 00:13:20 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 00:13:22 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 00:13:25 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 4: sync_c=4.828 +[May 1, 2026 - 00:13:25 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 00:13:27 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 00:13:30 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 5: sync_c=6.475 +[May 1, 2026 - 00:13:30 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 00:13:32 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 00:13:35 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 6: sync_c=5.886 +[May 1, 2026 - 00:13:35 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 00:13:37 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 00:13:40 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 7: sync_c=4.569 +[May 1, 2026 - 00:13:40 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 00:13:42 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 00:13:45 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 8: sync_c=5.338 +[May 1, 2026 - 00:13:45 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 00:13:47 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 00:13:50 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 9: sync_c=2.445 +[May 1, 2026 - 00:13:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging val0 stats at iteration 100-------------------- +[May 1, 2026 - 00:13:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0000 iter count: 10.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 1, 2026 - 00:14:07 | INFO | fastgen.callbacks.wandb:on_validation_end:763 ] Logged 10 val videos at iteration 100 +[May 1, 2026 - 00:14:07 | INFO | fastgen.callbacks.wandb:on_validation_end:769 ] [SyncEval] val0 mean sync_c: 5.116 (n=10) +[May 1, 2026 - 00:14:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.448837 41.612087 41.318428 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.223145 68.623047 68.089844 +util 92.750000 99.000000 89.000000 +[May 1, 2026 - 00:14:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 101 : avg iteration time 167.67 seconds +[May 1, 2026 - 00:14:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 101 : data loading time 0.00 +[May 1, 2026 - 00:14:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 101 : avg forward pass time 10.51 +[May 1, 2026 - 00:14:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 101 : backward pass time 10.13 +[May 1, 2026 - 00:14:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 101 : optimizer step time 1.57 +[May 1, 2026 - 00:14:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 101-------------------- +[May 1, 2026 - 00:14:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0082 iter count: 1.0 +[May 1, 2026 - 00:14:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[May 1, 2026 - 00:14:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:14:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 00:15:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.448589 41.611115 41.318405 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.232910 68.603516 68.109375 +util 96.000000 100.000000 89.000000 +[May 1, 2026 - 00:15:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 102 : avg iteration time 49.15 seconds +[May 1, 2026 - 00:15:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 102 : data loading time 0.00 +[May 1, 2026 - 00:15:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 102 : avg forward pass time 13.61 +[May 1, 2026 - 00:15:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 102 : backward pass time 10.19 +[May 1, 2026 - 00:15:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 102 : optimizer step time 1.57 +[May 1, 2026 - 00:15:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 102-------------------- +[May 1, 2026 - 00:15:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 00:15:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 00:15:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:15:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 00:16:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.448592 41.611122 41.318401 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.188965 68.603516 68.050781 +util 96.500000 100.000000 91.000000 +[May 1, 2026 - 00:16:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 103 : avg iteration time 43.74 seconds +[May 1, 2026 - 00:16:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 103 : data loading time 0.00 +[May 1, 2026 - 00:16:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 103 : avg forward pass time 10.90 +[May 1, 2026 - 00:16:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 103 : backward pass time 10.18 +[May 1, 2026 - 00:16:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 103 : optimizer step time 1.56 +[May 1, 2026 - 00:16:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 103-------------------- +[May 1, 2026 - 00:16:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0080 iter count: 1.0 +[May 1, 2026 - 00:16:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0080 iter count: 1.0 +[May 1, 2026 - 00:16:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:16:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.02 seconds +[May 1, 2026 - 00:17:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.448578 41.611027 41.318424 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.188965 68.603516 68.050781 +util 95.750000 99.000000 91.000000 +[May 1, 2026 - 00:17:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 104 : avg iteration time 46.39 seconds +[May 1, 2026 - 00:17:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 104 : data loading time 0.00 +[May 1, 2026 - 00:17:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 104 : avg forward pass time 12.23 +[May 1, 2026 - 00:17:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 104 : backward pass time 10.18 +[May 1, 2026 - 00:17:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 104 : optimizer step time 1.57 +[May 1, 2026 - 00:17:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 104-------------------- +[May 1, 2026 - 00:17:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0100 iter count: 1.0 +[May 1, 2026 - 00:17:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 1, 2026 - 00:17:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:17:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 00:19:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449060 41.608879 41.319416 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 135.959473 136.095703 135.914062 +util 69.000000 74.000000 64.000000 +[May 1, 2026 - 00:19:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 105 : avg iteration time 166.25 seconds +[May 1, 2026 - 00:19:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 105 : data loading time 0.00 +[May 1, 2026 - 00:19:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 105 : avg forward pass time 74.51 +[May 1, 2026 - 00:19:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 105 : backward pass time 8.43 +[May 1, 2026 - 00:19:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 105 : optimizer step time 0.39 +[May 1, 2026 - 00:19:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 105-------------------- +[May 1, 2026 - 00:19:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 150847.6345 iter count: 1.0 +[May 1, 2026 - 00:19:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7391 iter count: 1.0 +[May 1, 2026 - 00:19:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 150847.6345iter count: 1.0 +[May 1, 2026 - 00:19:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:19:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.0801iter count: 1.0 +[May 1, 2026 - 00:19:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.7999 iter count: 1.0 +[May 1, 2026 - 00:19:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.4376 iter count: 1.0 +[May 1, 2026 - 00:19:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 196709.8906iter count: 1.0 +[May 1, 2026 - 00:19:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 4.9516 iter count: 1.0 +[May 1, 2026 - 00:19:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 2885271.7500iter count: 1.0 +[May 1, 2026 - 00:19:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7391iter count: 1.0 +[May 1, 2026 - 00:19:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 6.4220 iter count: 1.0 +[May 1, 2026 - 00:19:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 378494.9375iter count: 1.0 +[May 1, 2026 - 00:19:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.0606 iter count: 1.0 +[May 1, 2026 - 00:19:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 24863.1816iter count: 1.0 +[May 1, 2026 - 00:19:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 7.4376 iter count: 1.0 +[May 1, 2026 - 00:19:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 2885271.7500iter count: 1.0 +[May 1, 2026 - 00:19:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 1.0050 iter count: 1.0 +[May 1, 2026 - 00:19:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 7.4637 iter count: 1.0 +[May 1, 2026 - 00:19:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[May 1, 2026 - 00:19:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:19:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 00:20:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449144 41.609161 41.319393 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.077148 68.324219 67.335938 +util 98.000000 100.000000 94.000000 +[May 1, 2026 - 00:20:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 106 : avg iteration time 45.20 seconds +[May 1, 2026 - 00:20:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 106 : data loading time 0.00 +[May 1, 2026 - 00:20:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 106 : avg forward pass time 11.67 +[May 1, 2026 - 00:20:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 106 : backward pass time 10.14 +[May 1, 2026 - 00:20:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 106 : optimizer step time 1.58 +[May 1, 2026 - 00:20:40 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 106-------------------- +[May 1, 2026 - 00:20:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0084 iter count: 1.0 +[May 1, 2026 - 00:20:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 1, 2026 - 00:20:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:20:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 00:21:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449153 41.609161 41.319393 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.735352 67.843750 67.699219 +util 94.000000 99.000000 88.000000 +[May 1, 2026 - 00:21:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 107 : avg iteration time 49.04 seconds +[May 1, 2026 - 00:21:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 107 : data loading time 0.00 +[May 1, 2026 - 00:21:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 107 : avg forward pass time 13.58 +[May 1, 2026 - 00:21:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 107 : backward pass time 10.15 +[May 1, 2026 - 00:21:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 107 : optimizer step time 1.57 +[May 1, 2026 - 00:21:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 107-------------------- +[May 1, 2026 - 00:21:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0085 iter count: 1.0 +[May 1, 2026 - 00:21:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 1, 2026 - 00:21:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:21:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 00:22:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449153 41.609161 41.319393 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.686523 67.882812 67.621094 +util 93.500000 100.000000 90.000000 +[May 1, 2026 - 00:22:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 108 : avg iteration time 49.03 seconds +[May 1, 2026 - 00:22:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 108 : data loading time 0.00 +[May 1, 2026 - 00:22:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 108 : avg forward pass time 13.58 +[May 1, 2026 - 00:22:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 108 : backward pass time 10.14 +[May 1, 2026 - 00:22:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 108 : optimizer step time 1.58 +[May 1, 2026 - 00:22:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 108-------------------- +[May 1, 2026 - 00:22:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0075 iter count: 1.0 +[May 1, 2026 - 00:22:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0075 iter count: 1.0 +[May 1, 2026 - 00:22:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:22:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 00:23:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449274 41.609646 41.319393 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.750000 67.835938 67.492188 +util 98.250000 100.000000 94.000000 +[May 1, 2026 - 00:23:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 109 : avg iteration time 46.32 seconds +[May 1, 2026 - 00:23:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 109 : data loading time 0.00 +[May 1, 2026 - 00:23:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 109 : avg forward pass time 12.23 +[May 1, 2026 - 00:23:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 109 : backward pass time 10.15 +[May 1, 2026 - 00:23:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 109 : optimizer step time 1.55 +[May 1, 2026 - 00:23:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 109-------------------- +[May 1, 2026 - 00:23:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0077 iter count: 1.0 +[May 1, 2026 - 00:23:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0077 iter count: 1.0 +[May 1, 2026 - 00:23:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:23:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 00:25:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449265 41.609406 41.319530 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.057129 136.134766 136.031250 +util 62.750000 70.000000 59.000000 +[May 1, 2026 - 00:25:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 110 : avg iteration time 166.09 seconds +[May 1, 2026 - 00:25:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 110 : data loading time 0.00 +[May 1, 2026 - 00:25:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 110 : avg forward pass time 74.46 +[May 1, 2026 - 00:25:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 110 : backward pass time 8.39 +[May 1, 2026 - 00:25:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 110 : optimizer step time 0.39 +[May 1, 2026 - 00:25:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 110-------------------- +[May 1, 2026 - 00:25:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 122387.9844 iter count: 1.0 +[May 1, 2026 - 00:25:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7196 iter count: 1.0 +[May 1, 2026 - 00:25:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 122387.9844iter count: 1.0 +[May 1, 2026 - 00:25:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:25:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.6705iter count: 1.0 +[May 1, 2026 - 00:25:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.9196 iter count: 1.0 +[May 1, 2026 - 00:25:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.0988 iter count: 1.0 +[May 1, 2026 - 00:25:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 188504.0938iter count: 1.0 +[May 1, 2026 - 00:25:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 46.4907iter count: 1.0 +[May 1, 2026 - 00:25:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 1465287.7500iter count: 1.0 +[May 1, 2026 - 00:25:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7196iter count: 1.0 +[May 1, 2026 - 00:25:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.9950 iter count: 1.0 +[May 1, 2026 - 00:25:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 21805.4648iter count: 1.0 +[May 1, 2026 - 00:25:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.4377 iter count: 1.0 +[May 1, 2026 - 00:25:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 7153.7705iter count: 1.0 +[May 1, 2026 - 00:25:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.3597 iter count: 1.0 +[May 1, 2026 - 00:25:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 828.3128iter count: 1.0 +[May 1, 2026 - 00:25:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.2517 iter count: 1.0 +[May 1, 2026 - 00:25:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 269251.9062iter count: 1.0 +[May 1, 2026 - 00:25:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[May 1, 2026 - 00:25:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:25:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 00:26:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449254 41.609406 41.319511 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.505859 67.542969 67.394531 +util 92.750000 99.000000 89.000000 +[May 1, 2026 - 00:26:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 111 : avg iteration time 47.79 seconds +[May 1, 2026 - 00:26:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 111 : data loading time 0.00 +[May 1, 2026 - 00:26:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 111 : avg forward pass time 12.97 +[May 1, 2026 - 00:26:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 111 : backward pass time 10.14 +[May 1, 2026 - 00:26:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 111 : optimizer step time 1.58 +[May 1, 2026 - 00:26:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 111-------------------- +[May 1, 2026 - 00:26:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0081 iter count: 1.0 +[May 1, 2026 - 00:26:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[May 1, 2026 - 00:26:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:26:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 00:27:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449345 41.609650 41.319523 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.481445 67.503906 67.414062 +util 96.500000 99.000000 92.000000 +[May 1, 2026 - 00:27:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 112 : avg iteration time 46.30 seconds +[May 1, 2026 - 00:27:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 112 : data loading time 0.00 +[May 1, 2026 - 00:27:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 112 : avg forward pass time 12.22 +[May 1, 2026 - 00:27:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 112 : backward pass time 10.15 +[May 1, 2026 - 00:27:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 112 : optimizer step time 1.56 +[May 1, 2026 - 00:27:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 112-------------------- +[May 1, 2026 - 00:27:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0086 iter count: 1.0 +[May 1, 2026 - 00:27:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 1, 2026 - 00:27:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:27:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 00:28:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449352 41.609650 41.319511 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.481445 67.503906 67.414062 +util 91.750000 96.000000 86.000000 +[May 1, 2026 - 00:28:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 113 : avg iteration time 43.54 seconds +[May 1, 2026 - 00:28:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 113 : data loading time 0.00 +[May 1, 2026 - 00:28:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 113 : avg forward pass time 10.85 +[May 1, 2026 - 00:28:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 113 : backward pass time 10.15 +[May 1, 2026 - 00:28:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 113 : optimizer step time 1.59 +[May 1, 2026 - 00:28:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 113-------------------- +[May 1, 2026 - 00:28:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0082 iter count: 1.0 +[May 1, 2026 - 00:28:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[May 1, 2026 - 00:28:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:28:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 00:28:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449359 41.609650 41.319511 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.471680 67.503906 67.375000 +util 93.250000 97.000000 89.000000 +[May 1, 2026 - 00:28:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 114 : avg iteration time 43.61 seconds +[May 1, 2026 - 00:28:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 114 : data loading time 0.00 +[May 1, 2026 - 00:28:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 114 : avg forward pass time 10.88 +[May 1, 2026 - 00:28:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 114 : backward pass time 10.15 +[May 1, 2026 - 00:28:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 114 : optimizer step time 1.57 +[May 1, 2026 - 00:28:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 114-------------------- +[May 1, 2026 - 00:28:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0083 iter count: 1.0 +[May 1, 2026 - 00:28:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[May 1, 2026 - 00:28:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:28:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 00:31:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449161 41.608624 41.319633 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.042480 136.050781 136.017578 +util 63.250000 72.000000 55.000000 +[May 1, 2026 - 00:31:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 115 : avg iteration time 168.52 seconds +[May 1, 2026 - 00:31:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 115 : data loading time 0.00 +[May 1, 2026 - 00:31:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 115 : avg forward pass time 75.67 +[May 1, 2026 - 00:31:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 115 : backward pass time 8.40 +[May 1, 2026 - 00:31:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 115 : optimizer step time 0.39 +[May 1, 2026 - 00:31:40 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 115-------------------- +[May 1, 2026 - 00:31:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 954375.3420 iter count: 1.0 +[May 1, 2026 - 00:31:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7002 iter count: 1.0 +[May 1, 2026 - 00:31:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 954375.3420iter count: 1.0 +[May 1, 2026 - 00:31:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:31:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.6644iter count: 1.0 +[May 1, 2026 - 00:31:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.5725 iter count: 1.0 +[May 1, 2026 - 00:31:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.6058 iter count: 1.0 +[May 1, 2026 - 00:31:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 1219702.3750iter count: 1.0 +[May 1, 2026 - 00:31:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 23.2189iter count: 1.0 +[May 1, 2026 - 00:31:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 29844802.0000iter count: 1.0 +[May 1, 2026 - 00:31:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7002iter count: 1.0 +[May 1, 2026 - 00:31:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 6.3783 iter count: 1.0 +[May 1, 2026 - 00:31:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 346850.4688iter count: 1.0 +[May 1, 2026 - 00:31:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 7.4699 iter count: 1.0 +[May 1, 2026 - 00:31:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 3078214.7500iter count: 1.0 +[May 1, 2026 - 00:31:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 2.3653 iter count: 1.0 +[May 1, 2026 - 00:31:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 113.3597iter count: 1.0 +[May 1, 2026 - 00:31:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 1.7706 iter count: 1.0 +[May 1, 2026 - 00:31:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 34.5085 iter count: 1.0 +[May 1, 2026 - 00:31:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 1, 2026 - 00:31:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:31:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 00:32:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449227 41.608891 41.319633 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.520020 67.542969 67.451172 +util 95.250000 99.000000 90.000000 +[May 1, 2026 - 00:32:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 116 : avg iteration time 50.51 seconds +[May 1, 2026 - 00:32:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 116 : data loading time 0.00 +[May 1, 2026 - 00:32:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 116 : avg forward pass time 14.33 +[May 1, 2026 - 00:32:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 116 : backward pass time 10.14 +[May 1, 2026 - 00:32:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 116 : optimizer step time 1.57 +[May 1, 2026 - 00:32:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 116-------------------- +[May 1, 2026 - 00:32:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0084 iter count: 1.0 +[May 1, 2026 - 00:32:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 1, 2026 - 00:32:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:32:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 00:33:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449259 41.608891 41.319740 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.466309 67.470703 67.464844 +util 93.750000 100.000000 90.000000 +[May 1, 2026 - 00:33:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 117 : avg iteration time 43.53 seconds +[May 1, 2026 - 00:33:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 117 : data loading time 0.00 +[May 1, 2026 - 00:33:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 117 : avg forward pass time 10.84 +[May 1, 2026 - 00:33:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 117 : backward pass time 10.14 +[May 1, 2026 - 00:33:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 117 : optimizer step time 1.57 +[May 1, 2026 - 00:33:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 117-------------------- +[May 1, 2026 - 00:33:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 00:33:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 00:33:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:33:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 00:33:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449033 41.607914 41.319740 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.500488 67.666016 67.445312 +util 95.750000 99.000000 89.000000 +[May 1, 2026 - 00:33:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 118 : avg iteration time 43.54 seconds +[May 1, 2026 - 00:33:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 118 : data loading time 0.00 +[May 1, 2026 - 00:33:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 118 : avg forward pass time 10.86 +[May 1, 2026 - 00:33:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 118 : backward pass time 10.14 +[May 1, 2026 - 00:33:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 118 : optimizer step time 1.56 +[May 1, 2026 - 00:33:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 118-------------------- +[May 1, 2026 - 00:33:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0085 iter count: 1.0 +[May 1, 2026 - 00:33:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 1, 2026 - 00:33:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:33:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 00:34:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449184 41.608437 41.319740 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.671387 67.718750 67.529297 +util 96.000000 100.000000 88.000000 +[May 1, 2026 - 00:34:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 119 : avg iteration time 46.39 seconds +[May 1, 2026 - 00:34:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 119 : data loading time 0.00 +[May 1, 2026 - 00:34:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 119 : avg forward pass time 12.27 +[May 1, 2026 - 00:34:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 119 : backward pass time 10.14 +[May 1, 2026 - 00:34:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 119 : optimizer step time 1.58 +[May 1, 2026 - 00:34:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 119-------------------- +[May 1, 2026 - 00:34:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 00:34:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 00:34:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:34:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 00:37:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.448936 41.607258 41.319878 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.091309 136.212891 136.050781 +util 68.750000 73.000000 64.000000 +[May 1, 2026 - 00:37:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 120 : avg iteration time 168.79 seconds +[May 1, 2026 - 00:37:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 120 : data loading time 0.00 +[May 1, 2026 - 00:37:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 120 : avg forward pass time 75.77 +[May 1, 2026 - 00:37:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 120 : backward pass time 8.44 +[May 1, 2026 - 00:37:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 120 : optimizer step time 0.39 +[May 1, 2026 - 00:37:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 120-------------------- +[May 1, 2026 - 00:37:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 845732.2268 iter count: 1.0 +[May 1, 2026 - 00:37:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7448 iter count: 1.0 +[May 1, 2026 - 00:37:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 845732.2268iter count: 1.0 +[May 1, 2026 - 00:37:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:37:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.5158iter count: 1.0 +[May 1, 2026 - 00:37:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.8677 iter count: 1.0 +[May 1, 2026 - 00:37:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.6659 iter count: 1.0 +[May 1, 2026 - 00:37:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 1152621.1250iter count: 1.0 +[May 1, 2026 - 00:37:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 5.6713 iter count: 1.0 +[May 1, 2026 - 00:37:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 33658356.0000iter count: 1.0 +[May 1, 2026 - 00:37:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7448iter count: 1.0 +[May 1, 2026 - 00:37:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.3434 iter count: 1.0 +[May 1, 2026 - 00:37:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 5924.5000iter count: 1.0 +[May 1, 2026 - 00:37:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 2.2253 iter count: 1.0 +[May 1, 2026 - 00:37:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 85.6747 iter count: 1.0 +[May 1, 2026 - 00:37:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.3686 iter count: 1.0 +[May 1, 2026 - 00:37:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 6230.6982iter count: 1.0 +[May 1, 2026 - 00:37:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.9872 iter count: 1.0 +[May 1, 2026 - 00:37:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 2905.7900iter count: 1.0 +[May 1, 2026 - 00:37:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[May 1, 2026 - 00:37:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:37:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 00:38:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449016 41.607563 41.319878 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.681152 68.154297 67.523438 +util 97.750000 100.000000 93.000000 +[May 1, 2026 - 00:38:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 121 : avg iteration time 45.17 seconds +[May 1, 2026 - 00:38:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 121 : data loading time 0.00 +[May 1, 2026 - 00:38:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 121 : avg forward pass time 11.65 +[May 1, 2026 - 00:38:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 121 : backward pass time 10.14 +[May 1, 2026 - 00:38:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 121 : optimizer step time 1.57 +[May 1, 2026 - 00:38:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 121-------------------- +[May 1, 2026 - 00:38:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 00:38:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 00:38:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:38:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 00:39:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449265 41.608532 41.319881 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.578613 68.095703 67.406250 +util 92.000000 100.000000 87.000000 +[May 1, 2026 - 00:39:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 122 : avg iteration time 46.28 seconds +[May 1, 2026 - 00:39:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 122 : data loading time 0.00 +[May 1, 2026 - 00:39:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 122 : avg forward pass time 12.20 +[May 1, 2026 - 00:39:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 122 : backward pass time 10.16 +[May 1, 2026 - 00:39:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 122 : optimizer step time 1.58 +[May 1, 2026 - 00:39:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 122-------------------- +[May 1, 2026 - 00:39:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0085 iter count: 1.0 +[May 1, 2026 - 00:39:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 1, 2026 - 00:39:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:39:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 00:39:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449278 41.608578 41.319881 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.397949 68.076172 67.171875 +util 95.750000 100.000000 88.000000 +[May 1, 2026 - 00:39:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 123 : avg iteration time 46.56 seconds +[May 1, 2026 - 00:39:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 123 : data loading time 0.00 +[May 1, 2026 - 00:39:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 123 : avg forward pass time 12.35 +[May 1, 2026 - 00:39:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 123 : backward pass time 10.13 +[May 1, 2026 - 00:39:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 123 : optimizer step time 1.58 +[May 1, 2026 - 00:39:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 123-------------------- +[May 1, 2026 - 00:39:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 1, 2026 - 00:39:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 1, 2026 - 00:39:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:39:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 00:40:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449295 41.608532 41.319881 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.500488 67.503906 67.490234 +util 98.500000 100.000000 95.000000 +[May 1, 2026 - 00:40:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 124 : avg iteration time 46.39 seconds +[May 1, 2026 - 00:40:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 124 : data loading time 0.00 +[May 1, 2026 - 00:40:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 124 : avg forward pass time 12.25 +[May 1, 2026 - 00:40:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 124 : backward pass time 10.17 +[May 1, 2026 - 00:40:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 124 : optimizer step time 1.57 +[May 1, 2026 - 00:40:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 124-------------------- +[May 1, 2026 - 00:40:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 00:40:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 00:40:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:40:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 00:43:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449045 41.607121 41.319973 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.013184 136.076172 135.992188 +util 64.250000 72.000000 60.000000 +[May 1, 2026 - 00:43:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 125 : avg iteration time 163.10 seconds +[May 1, 2026 - 00:43:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 125 : data loading time 0.00 +[May 1, 2026 - 00:43:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 125 : avg forward pass time 72.95 +[May 1, 2026 - 00:43:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 125 : backward pass time 8.42 +[May 1, 2026 - 00:43:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 125 : optimizer step time 0.39 +[May 1, 2026 - 00:43:21 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 125-------------------- +[May 1, 2026 - 00:43:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 9224010.8628 iter count: 1.0 +[May 1, 2026 - 00:43:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6253 iter count: 1.0 +[May 1, 2026 - 00:43:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 9224010.8628iter count: 1.0 +[May 1, 2026 - 00:43:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:43:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.4985iter count: 1.0 +[May 1, 2026 - 00:43:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.1171 iter count: 1.0 +[May 1, 2026 - 00:43:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 10.0581iter count: 1.0 +[May 1, 2026 - 00:43:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 17384764.0000iter count: 1.0 +[May 1, 2026 - 00:43:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 9.3397 iter count: 1.0 +[May 1, 2026 - 00:43:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 544919104.0000iter count: 1.0 +[May 1, 2026 - 00:43:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6253iter count: 1.0 +[May 1, 2026 - 00:43:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.0611 iter count: 1.0 +[May 1, 2026 - 00:43:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 3368.3655iter count: 1.0 +[May 1, 2026 - 00:43:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.7523 iter count: 1.0 +[May 1, 2026 - 00:43:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 1816.2271iter count: 1.0 +[May 1, 2026 - 00:43:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 6.0513 iter count: 1.0 +[May 1, 2026 - 00:43:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 180350.1250iter count: 1.0 +[May 1, 2026 - 00:43:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.6017 iter count: 1.0 +[May 1, 2026 - 00:43:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 73374.7188iter count: 1.0 +[May 1, 2026 - 00:43:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[May 1, 2026 - 00:43:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:43:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 00:44:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449108 41.607368 41.319973 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.754883 68.039062 67.660156 +util 92.250000 99.000000 88.000000 +[May 1, 2026 - 00:44:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 126 : avg iteration time 50.57 seconds +[May 1, 2026 - 00:44:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 126 : data loading time 0.00 +[May 1, 2026 - 00:44:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 126 : avg forward pass time 14.37 +[May 1, 2026 - 00:44:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 126 : backward pass time 10.12 +[May 1, 2026 - 00:44:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 126 : optimizer step time 1.57 +[May 1, 2026 - 00:44:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 126-------------------- +[May 1, 2026 - 00:44:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 1, 2026 - 00:44:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 1, 2026 - 00:44:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:44:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 00:44:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449353 41.608364 41.319973 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.769531 67.980469 67.699219 +util 96.500000 100.000000 89.000000 +[May 1, 2026 - 00:44:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 127 : avg iteration time 43.56 seconds +[May 1, 2026 - 00:44:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 127 : data loading time 0.00 +[May 1, 2026 - 00:44:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 127 : avg forward pass time 10.85 +[May 1, 2026 - 00:44:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 127 : backward pass time 10.15 +[May 1, 2026 - 00:44:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 127 : optimizer step time 1.56 +[May 1, 2026 - 00:44:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 127-------------------- +[May 1, 2026 - 00:44:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0085 iter count: 1.0 +[May 1, 2026 - 00:44:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 1, 2026 - 00:44:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:44:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 00:45:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449389 41.608410 41.320015 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.842773 67.980469 67.796875 +util 93.500000 99.000000 90.000000 +[May 1, 2026 - 00:45:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 128 : avg iteration time 43.65 seconds +[May 1, 2026 - 00:45:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 128 : data loading time 0.00 +[May 1, 2026 - 00:45:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 128 : avg forward pass time 10.89 +[May 1, 2026 - 00:45:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 128 : backward pass time 10.15 +[May 1, 2026 - 00:45:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 128 : optimizer step time 1.59 +[May 1, 2026 - 00:45:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 128-------------------- +[May 1, 2026 - 00:45:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0083 iter count: 1.0 +[May 1, 2026 - 00:45:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[May 1, 2026 - 00:45:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:45:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 00:46:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449367 41.608368 41.320015 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.720703 68.019531 67.621094 +util 93.500000 100.000000 89.000000 +[May 1, 2026 - 00:46:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 129 : avg iteration time 43.60 seconds +[May 1, 2026 - 00:46:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 129 : data loading time 0.00 +[May 1, 2026 - 00:46:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 129 : avg forward pass time 10.88 +[May 1, 2026 - 00:46:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 129 : backward pass time 10.13 +[May 1, 2026 - 00:46:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 129 : optimizer step time 1.57 +[May 1, 2026 - 00:46:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 129-------------------- +[May 1, 2026 - 00:46:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 1, 2026 - 00:46:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 1, 2026 - 00:46:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:46:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 00:49:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449244 41.607674 41.320038 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.011719 136.076172 135.990234 +util 68.250000 73.000000 63.000000 +[May 1, 2026 - 00:49:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 130 : avg iteration time 165.73 seconds +[May 1, 2026 - 00:49:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 130 : data loading time 0.00 +[May 1, 2026 - 00:49:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 130 : avg forward pass time 74.26 +[May 1, 2026 - 00:49:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 130 : backward pass time 8.43 +[May 1, 2026 - 00:49:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 130 : optimizer step time 0.39 +[May 1, 2026 - 00:49:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 130-------------------- +[May 1, 2026 - 00:49:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 301358.9490 iter count: 1.0 +[May 1, 2026 - 00:49:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6975 iter count: 1.0 +[May 1, 2026 - 00:49:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 301358.9490iter count: 1.0 +[May 1, 2026 - 00:49:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:49:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.3539iter count: 1.0 +[May 1, 2026 - 00:49:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.2726 iter count: 1.0 +[May 1, 2026 - 00:49:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.8047 iter count: 1.0 +[May 1, 2026 - 00:49:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 452218.3438iter count: 1.0 +[May 1, 2026 - 00:49:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 12.7458iter count: 1.0 +[May 1, 2026 - 00:49:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 6012219.0000iter count: 1.0 +[May 1, 2026 - 00:49:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6975iter count: 1.0 +[May 1, 2026 - 00:49:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.0029 iter count: 1.0 +[May 1, 2026 - 00:49:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 22152.8203iter count: 1.0 +[May 1, 2026 - 00:49:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.6185 iter count: 1.0 +[May 1, 2026 - 00:49:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 75887.2031iter count: 1.0 +[May 1, 2026 - 00:49:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 1.3454 iter count: 1.0 +[May 1, 2026 - 00:49:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 14.7428 iter count: 1.0 +[May 1, 2026 - 00:49:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.1348 iter count: 1.0 +[May 1, 2026 - 00:49:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 213130.7188iter count: 1.0 +[May 1, 2026 - 00:49:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 1, 2026 - 00:49:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:49:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 00:49:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449448 41.608410 41.320042 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.018555 68.156250 67.972656 +util 96.250000 100.000000 89.000000 +[May 1, 2026 - 00:49:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 131 : avg iteration time 50.81 seconds +[May 1, 2026 - 00:49:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 131 : data loading time 0.00 +[May 1, 2026 - 00:49:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 131 : avg forward pass time 14.47 +[May 1, 2026 - 00:49:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 131 : backward pass time 10.15 +[May 1, 2026 - 00:49:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 131 : optimizer step time 1.57 +[May 1, 2026 - 00:49:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 131-------------------- +[May 1, 2026 - 00:49:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0108 iter count: 1.0 +[May 1, 2026 - 00:49:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 1, 2026 - 00:49:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:49:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 00:50:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450127 41.611065 41.320061 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.662109 68.136719 67.503906 +util 94.000000 100.000000 90.000000 +[May 1, 2026 - 00:50:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 132 : avg iteration time 46.27 seconds +[May 1, 2026 - 00:50:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 132 : data loading time 0.00 +[May 1, 2026 - 00:50:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 132 : avg forward pass time 12.20 +[May 1, 2026 - 00:50:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 132 : backward pass time 10.15 +[May 1, 2026 - 00:50:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 132 : optimizer step time 1.57 +[May 1, 2026 - 00:50:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 132-------------------- +[May 1, 2026 - 00:50:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 1, 2026 - 00:50:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 1, 2026 - 00:50:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:50:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 00:51:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450182 41.611301 41.320084 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.652344 68.156250 67.484375 +util 93.750000 98.000000 89.000000 +[May 1, 2026 - 00:51:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 133 : avg iteration time 46.24 seconds +[May 1, 2026 - 00:51:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 133 : data loading time 0.00 +[May 1, 2026 - 00:51:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 133 : avg forward pass time 12.21 +[May 1, 2026 - 00:51:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 133 : backward pass time 10.12 +[May 1, 2026 - 00:51:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 133 : optimizer step time 1.57 +[May 1, 2026 - 00:51:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 133-------------------- +[May 1, 2026 - 00:51:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 00:51:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 00:51:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:51:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 00:52:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450187 41.611324 41.320061 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.642578 68.117188 67.484375 +util 95.500000 99.000000 88.000000 +[May 1, 2026 - 00:52:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 134 : avg iteration time 49.09 seconds +[May 1, 2026 - 00:52:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 134 : data loading time 0.00 +[May 1, 2026 - 00:52:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 134 : avg forward pass time 13.60 +[May 1, 2026 - 00:52:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 134 : backward pass time 10.16 +[May 1, 2026 - 00:52:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 134 : optimizer step time 1.57 +[May 1, 2026 - 00:52:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 134-------------------- +[May 1, 2026 - 00:52:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 00:52:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 00:52:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:52:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 00:55:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449629 41.608994 41.320179 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.027832 136.134766 135.992188 +util 65.000000 69.000000 61.000000 +[May 1, 2026 - 00:55:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 135 : avg iteration time 163.55 seconds +[May 1, 2026 - 00:55:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 135 : data loading time 0.00 +[May 1, 2026 - 00:55:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 135 : avg forward pass time 73.16 +[May 1, 2026 - 00:55:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 135 : backward pass time 8.45 +[May 1, 2026 - 00:55:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 135 : optimizer step time 0.39 +[May 1, 2026 - 00:55:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 135-------------------- +[May 1, 2026 - 00:55:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 90000.0908 iter count: 1.0 +[May 1, 2026 - 00:55:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6926 iter count: 1.0 +[May 1, 2026 - 00:55:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 90000.0908iter count: 1.0 +[May 1, 2026 - 00:55:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:55:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 3.9213iter count: 1.0 +[May 1, 2026 - 00:55:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.7897 iter count: 1.0 +[May 1, 2026 - 00:55:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.4810 iter count: 1.0 +[May 1, 2026 - 00:55:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 135411.1094iter count: 1.0 +[May 1, 2026 - 00:55:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 4.8519 iter count: 1.0 +[May 1, 2026 - 00:55:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 3147371.2500iter count: 1.0 +[May 1, 2026 - 00:55:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6926iter count: 1.0 +[May 1, 2026 - 00:55:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.5098 iter count: 1.0 +[May 1, 2026 - 00:55:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 8262.8877iter count: 1.0 +[May 1, 2026 - 00:55:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 1.9159 iter count: 1.0 +[May 1, 2026 - 00:55:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 46.1426 iter count: 1.0 +[May 1, 2026 - 00:55:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.9004 iter count: 1.0 +[May 1, 2026 - 00:55:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 2442.7798iter count: 1.0 +[May 1, 2026 - 00:55:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 2.1506 iter count: 1.0 +[May 1, 2026 - 00:55:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 73.7912 iter count: 1.0 +[May 1, 2026 - 00:55:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 1, 2026 - 00:55:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:55:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 00:55:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449651 41.609058 41.320202 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.569336 67.941406 67.445312 +util 95.500000 100.000000 88.000000 +[May 1, 2026 - 00:55:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 136 : avg iteration time 47.85 seconds +[May 1, 2026 - 00:55:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 136 : data loading time 0.00 +[May 1, 2026 - 00:55:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 136 : avg forward pass time 13.02 +[May 1, 2026 - 00:55:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 136 : backward pass time 10.14 +[May 1, 2026 - 00:55:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 136 : optimizer step time 1.56 +[May 1, 2026 - 00:55:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 136-------------------- +[May 1, 2026 - 00:55:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0085 iter count: 1.0 +[May 1, 2026 - 00:55:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 1, 2026 - 00:55:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:55:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 00:56:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449795 41.609634 41.320202 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.579102 67.921875 67.464844 +util 94.000000 99.000000 88.000000 +[May 1, 2026 - 00:56:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 137 : avg iteration time 46.24 seconds +[May 1, 2026 - 00:56:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 137 : data loading time 0.00 +[May 1, 2026 - 00:56:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 137 : avg forward pass time 12.19 +[May 1, 2026 - 00:56:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 137 : backward pass time 10.14 +[May 1, 2026 - 00:56:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 137 : optimizer step time 1.58 +[May 1, 2026 - 00:56:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 137-------------------- +[May 1, 2026 - 00:56:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 00:56:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 00:56:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:56:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 00:57:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449806 41.609680 41.320202 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.344727 67.921875 67.152344 +util 92.500000 99.000000 88.000000 +[May 1, 2026 - 00:57:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 138 : avg iteration time 46.22 seconds +[May 1, 2026 - 00:57:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 138 : data loading time 0.00 +[May 1, 2026 - 00:57:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 138 : avg forward pass time 12.19 +[May 1, 2026 - 00:57:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 138 : backward pass time 10.12 +[May 1, 2026 - 00:57:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 138 : optimizer step time 1.58 +[May 1, 2026 - 00:57:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 138-------------------- +[May 1, 2026 - 00:57:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0082 iter count: 1.0 +[May 1, 2026 - 00:57:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[May 1, 2026 - 00:57:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:57:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 00:58:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449837 41.609680 41.320301 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.608398 67.921875 67.503906 +util 92.750000 97.000000 88.000000 +[May 1, 2026 - 00:58:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 139 : avg iteration time 46.25 seconds +[May 1, 2026 - 00:58:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 139 : data loading time 0.00 +[May 1, 2026 - 00:58:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 139 : avg forward pass time 12.19 +[May 1, 2026 - 00:58:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 139 : backward pass time 10.14 +[May 1, 2026 - 00:58:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 139 : optimizer step time 1.58 +[May 1, 2026 - 00:58:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 139-------------------- +[May 1, 2026 - 00:58:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 00:58:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 00:58:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 00:58:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 01:00:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449436 41.607971 41.320431 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.011719 136.076172 135.990234 +util 68.000000 72.000000 64.000000 +[May 1, 2026 - 01:00:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 140 : avg iteration time 168.68 seconds +[May 1, 2026 - 01:00:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 140 : data loading time 0.00 +[May 1, 2026 - 01:00:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 140 : avg forward pass time 75.74 +[May 1, 2026 - 01:00:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 140 : backward pass time 8.41 +[May 1, 2026 - 01:00:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 140 : optimizer step time 0.40 +[May 1, 2026 - 01:00:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 140-------------------- +[May 1, 2026 - 01:00:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 43940.2158 iter count: 1.0 +[May 1, 2026 - 01:00:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7014 iter count: 1.0 +[May 1, 2026 - 01:00:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 43940.2158iter count: 1.0 +[May 1, 2026 - 01:00:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:00:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.2147iter count: 1.0 +[May 1, 2026 - 01:00:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.8837 iter count: 1.0 +[May 1, 2026 - 01:00:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 6.4660 iter count: 1.0 +[May 1, 2026 - 01:00:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 65279.3047iter count: 1.0 +[May 1, 2026 - 01:00:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 5.8560 iter count: 1.0 +[May 1, 2026 - 01:00:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 413319.1875iter count: 1.0 +[May 1, 2026 - 01:00:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7014iter count: 1.0 +[May 1, 2026 - 01:00:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.1680 iter count: 1.0 +[May 1, 2026 - 01:00:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 30822.0957iter count: 1.0 +[May 1, 2026 - 01:00:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.9723 iter count: 1.0 +[May 1, 2026 - 01:00:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 20838.4316iter count: 1.0 +[May 1, 2026 - 01:00:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.9866 iter count: 1.0 +[May 1, 2026 - 01:00:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 158437.5625iter count: 1.0 +[May 1, 2026 - 01:00:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.8513 iter count: 1.0 +[May 1, 2026 - 01:00:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 2213.9419iter count: 1.0 +[May 1, 2026 - 01:00:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[May 1, 2026 - 01:00:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:00:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 01:01:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449484 41.608173 41.320412 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.446777 67.464844 67.392578 +util 97.250000 99.000000 93.000000 +[May 1, 2026 - 01:01:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 141 : avg iteration time 47.79 seconds +[May 1, 2026 - 01:01:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 141 : data loading time 0.00 +[May 1, 2026 - 01:01:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 141 : avg forward pass time 12.97 +[May 1, 2026 - 01:01:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 141 : backward pass time 10.13 +[May 1, 2026 - 01:01:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 141 : optimizer step time 1.57 +[May 1, 2026 - 01:01:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 141-------------------- +[May 1, 2026 - 01:01:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 1, 2026 - 01:01:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 1, 2026 - 01:01:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:01:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 01:02:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449907 41.609821 41.320431 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.393066 67.822266 67.250000 +util 96.250000 100.000000 91.000000 +[May 1, 2026 - 01:02:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 142 : avg iteration time 46.17 seconds +[May 1, 2026 - 01:02:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 142 : data loading time 0.00 +[May 1, 2026 - 01:02:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 142 : avg forward pass time 12.17 +[May 1, 2026 - 01:02:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 142 : backward pass time 10.11 +[May 1, 2026 - 01:02:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 142 : optimizer step time 1.58 +[May 1, 2026 - 01:02:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 142-------------------- +[May 1, 2026 - 01:02:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 01:02:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 01:02:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:02:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 01:03:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449914 41.609848 41.320431 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.695801 68.212891 67.523438 +util 95.750000 100.000000 89.000000 +[May 1, 2026 - 01:03:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 143 : avg iteration time 46.25 seconds +[May 1, 2026 - 01:03:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 143 : data loading time 0.00 +[May 1, 2026 - 01:03:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 143 : avg forward pass time 12.19 +[May 1, 2026 - 01:03:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 143 : backward pass time 10.14 +[May 1, 2026 - 01:03:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 143 : optimizer step time 1.58 +[May 1, 2026 - 01:03:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 143-------------------- +[May 1, 2026 - 01:03:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0084 iter count: 1.0 +[May 1, 2026 - 01:03:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 1, 2026 - 01:03:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:03:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 01:04:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449981 41.610104 41.320431 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.520020 67.626953 67.484375 +util 95.500000 99.000000 88.000000 +[May 1, 2026 - 01:04:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 144 : avg iteration time 43.61 seconds +[May 1, 2026 - 01:04:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 144 : data loading time 0.00 +[May 1, 2026 - 01:04:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 144 : avg forward pass time 10.86 +[May 1, 2026 - 01:04:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 144 : backward pass time 10.14 +[May 1, 2026 - 01:04:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 144 : optimizer step time 1.58 +[May 1, 2026 - 01:04:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 144-------------------- +[May 1, 2026 - 01:04:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0083 iter count: 1.0 +[May 1, 2026 - 01:04:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[May 1, 2026 - 01:04:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:04:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 01:06:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449409 41.607841 41.320408 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.032715 136.037109 136.031250 +util 64.500000 72.000000 57.000000 +[May 1, 2026 - 01:06:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 145 : avg iteration time 165.84 seconds +[May 1, 2026 - 01:06:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 145 : data loading time 0.00 +[May 1, 2026 - 01:06:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 145 : avg forward pass time 74.32 +[May 1, 2026 - 01:06:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 145 : backward pass time 8.40 +[May 1, 2026 - 01:06:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 145 : optimizer step time 0.39 +[May 1, 2026 - 01:06:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 145-------------------- +[May 1, 2026 - 01:06:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 26201.3918 iter count: 1.0 +[May 1, 2026 - 01:06:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7048 iter count: 1.0 +[May 1, 2026 - 01:06:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 26201.3918iter count: 1.0 +[May 1, 2026 - 01:06:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:06:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.4761iter count: 1.0 +[May 1, 2026 - 01:06:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 2.2250 iter count: 1.0 +[May 1, 2026 - 01:06:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 6.1208 iter count: 1.0 +[May 1, 2026 - 01:06:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 36183.1992iter count: 1.0 +[May 1, 2026 - 01:06:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 85.6308iter count: 1.0 +[May 1, 2026 - 01:06:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 207216.0156iter count: 1.0 +[May 1, 2026 - 01:06:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7048iter count: 1.0 +[May 1, 2026 - 01:06:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.6175 iter count: 1.0 +[May 1, 2026 - 01:06:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 75734.0000iter count: 1.0 +[May 1, 2026 - 01:06:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 2.2250 iter count: 1.0 +[May 1, 2026 - 01:06:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 85.6308 iter count: 1.0 +[May 1, 2026 - 01:06:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.5415 iter count: 1.0 +[May 1, 2026 - 01:06:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 8804.6436iter count: 1.0 +[May 1, 2026 - 01:06:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.6612 iter count: 1.0 +[May 1, 2026 - 01:06:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 1513.7766iter count: 1.0 +[May 1, 2026 - 01:06:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0075 iter count: 1.0 +[May 1, 2026 - 01:06:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:06:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 01:07:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449428 41.607834 41.320408 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.457031 67.484375 67.375000 +util 93.500000 99.000000 87.000000 +[May 1, 2026 - 01:07:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 146 : avg iteration time 47.78 seconds +[May 1, 2026 - 01:07:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 146 : data loading time 0.00 +[May 1, 2026 - 01:07:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 146 : avg forward pass time 12.98 +[May 1, 2026 - 01:07:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 146 : backward pass time 10.12 +[May 1, 2026 - 01:07:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 146 : optimizer step time 1.57 +[May 1, 2026 - 01:07:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 146-------------------- +[May 1, 2026 - 01:07:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0104 iter count: 1.0 +[May 1, 2026 - 01:07:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 1, 2026 - 01:07:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:07:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 01:08:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449805 41.609283 41.320431 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.574219 67.640625 67.375000 +util 93.500000 100.000000 88.000000 +[May 1, 2026 - 01:08:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 147 : avg iteration time 48.99 seconds +[May 1, 2026 - 01:08:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 147 : data loading time 0.00 +[May 1, 2026 - 01:08:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 147 : avg forward pass time 13.56 +[May 1, 2026 - 01:08:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 147 : backward pass time 10.14 +[May 1, 2026 - 01:08:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 147 : optimizer step time 1.58 +[May 1, 2026 - 01:08:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 147-------------------- +[May 1, 2026 - 01:08:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 01:08:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 01:08:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:08:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 01:09:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449800 41.609283 41.320408 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.603516 67.621094 67.550781 +util 93.750000 98.000000 89.000000 +[May 1, 2026 - 01:09:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 148 : avg iteration time 46.10 seconds +[May 1, 2026 - 01:09:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 148 : data loading time 0.00 +[May 1, 2026 - 01:09:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 148 : avg forward pass time 12.15 +[May 1, 2026 - 01:09:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 148 : backward pass time 10.10 +[May 1, 2026 - 01:09:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 148 : optimizer step time 1.57 +[May 1, 2026 - 01:09:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 148-------------------- +[May 1, 2026 - 01:09:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0076 iter count: 1.0 +[May 1, 2026 - 01:09:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0076 iter count: 1.0 +[May 1, 2026 - 01:09:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:09:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 01:09:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449801 41.609291 41.320408 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.183594 67.453125 67.093750 +util 94.750000 99.000000 90.000000 +[May 1, 2026 - 01:09:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 149 : avg iteration time 43.62 seconds +[May 1, 2026 - 01:09:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 149 : data loading time 0.00 +[May 1, 2026 - 01:09:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 149 : avg forward pass time 10.87 +[May 1, 2026 - 01:09:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 149 : backward pass time 10.15 +[May 1, 2026 - 01:09:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 149 : optimizer step time 1.58 +[May 1, 2026 - 01:09:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 149-------------------- +[May 1, 2026 - 01:09:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0085 iter count: 1.0 +[May 1, 2026 - 01:09:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 1, 2026 - 01:09:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:09:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 01:12:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449342 41.607380 41.320480 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.021484 136.056641 136.009766 +util 62.500000 70.000000 55.000000 +[May 1, 2026 - 01:12:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 150 : avg iteration time 166.29 seconds +[May 1, 2026 - 01:12:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 150 : data loading time 0.00 +[May 1, 2026 - 01:12:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 150 : avg forward pass time 74.53 +[May 1, 2026 - 01:12:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 150 : backward pass time 8.43 +[May 1, 2026 - 01:12:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 150 : optimizer step time 0.39 +[May 1, 2026 - 01:12:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 150-------------------- +[May 1, 2026 - 01:12:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 137192.5986 iter count: 1.0 +[May 1, 2026 - 01:12:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6870 iter count: 1.0 +[May 1, 2026 - 01:12:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 137192.5986iter count: 1.0 +[May 1, 2026 - 01:12:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:12:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.6105iter count: 1.0 +[May 1, 2026 - 01:12:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.1327 iter count: 1.0 +[May 1, 2026 - 01:12:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.3876 iter count: 1.0 +[May 1, 2026 - 01:12:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 195912.1719iter count: 1.0 +[May 1, 2026 - 01:12:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 9.6353 iter count: 1.0 +[May 1, 2026 - 01:12:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 2610818.0000iter count: 1.0 +[May 1, 2026 - 01:12:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6870iter count: 1.0 +[May 1, 2026 - 01:12:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 2.7473 iter count: 1.0 +[May 1, 2026 - 01:12:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 243.3501iter count: 1.0 +[May 1, 2026 - 01:12:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.9305 iter count: 1.0 +[May 1, 2026 - 01:12:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 141630.5469iter count: 1.0 +[May 1, 2026 - 01:12:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 6.6407 iter count: 1.0 +[May 1, 2026 - 01:12:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 586194.1875iter count: 1.0 +[May 1, 2026 - 01:12:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.1801 iter count: 1.0 +[May 1, 2026 - 01:12:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 4273.9526iter count: 1.0 +[May 1, 2026 - 01:12:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 1, 2026 - 01:12:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:12:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 01:13:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449452 41.607822 41.320480 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.476074 67.568359 67.445312 +util 94.250000 100.000000 90.000000 +[May 1, 2026 - 01:13:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 151 : avg iteration time 47.87 seconds +[May 1, 2026 - 01:13:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 151 : data loading time 0.00 +[May 1, 2026 - 01:13:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 151 : avg forward pass time 12.98 +[May 1, 2026 - 01:13:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 151 : backward pass time 10.17 +[May 1, 2026 - 01:13:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 151 : optimizer step time 1.57 +[May 1, 2026 - 01:13:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 151-------------------- +[May 1, 2026 - 01:13:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 1, 2026 - 01:13:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 1, 2026 - 01:13:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:13:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 01:14:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449919 41.609676 41.320480 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.637207 67.679688 67.509766 +util 91.750000 99.000000 87.000000 +[May 1, 2026 - 01:14:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 152 : avg iteration time 48.97 seconds +[May 1, 2026 - 01:14:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 152 : data loading time 0.00 +[May 1, 2026 - 01:14:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 152 : avg forward pass time 13.55 +[May 1, 2026 - 01:14:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 152 : backward pass time 10.14 +[May 1, 2026 - 01:14:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 152 : optimizer step time 1.58 +[May 1, 2026 - 01:14:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 152-------------------- +[May 1, 2026 - 01:14:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 01:14:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 01:14:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:14:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 01:15:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449919 41.609676 41.320480 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.529785 67.542969 67.490234 +util 92.500000 99.000000 88.000000 +[May 1, 2026 - 01:15:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 153 : avg iteration time 46.23 seconds +[May 1, 2026 - 01:15:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 153 : data loading time 0.00 +[May 1, 2026 - 01:15:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 153 : avg forward pass time 12.19 +[May 1, 2026 - 01:15:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 153 : backward pass time 10.12 +[May 1, 2026 - 01:15:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 153 : optimizer step time 1.58 +[May 1, 2026 - 01:15:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 153-------------------- +[May 1, 2026 - 01:15:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0081 iter count: 1.0 +[May 1, 2026 - 01:15:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[May 1, 2026 - 01:15:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:15:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 01:15:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449928 41.609692 41.320480 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.490723 67.568359 67.464844 +util 94.750000 99.000000 90.000000 +[May 1, 2026 - 01:15:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 154 : avg iteration time 43.65 seconds +[May 1, 2026 - 01:15:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 154 : data loading time 0.00 +[May 1, 2026 - 01:15:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 154 : avg forward pass time 10.91 +[May 1, 2026 - 01:15:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 154 : backward pass time 10.13 +[May 1, 2026 - 01:15:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 154 : optimizer step time 1.56 +[May 1, 2026 - 01:15:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 154-------------------- +[May 1, 2026 - 01:15:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0085 iter count: 1.0 +[May 1, 2026 - 01:15:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 1, 2026 - 01:15:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:15:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 01:18:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449438 41.607498 41.320599 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.008301 136.056641 135.992188 +util 65.250000 73.000000 61.000000 +[May 1, 2026 - 01:18:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 155 : avg iteration time 163.20 seconds +[May 1, 2026 - 01:18:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 155 : data loading time 0.00 +[May 1, 2026 - 01:18:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 155 : avg forward pass time 73.01 +[May 1, 2026 - 01:18:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 155 : backward pass time 8.38 +[May 1, 2026 - 01:18:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 155 : optimizer step time 0.41 +[May 1, 2026 - 01:18:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 155-------------------- +[May 1, 2026 - 01:18:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 621258.4249 iter count: 1.0 +[May 1, 2026 - 01:18:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6573 iter count: 1.0 +[May 1, 2026 - 01:18:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 621258.4249iter count: 1.0 +[May 1, 2026 - 01:18:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:18:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.7627iter count: 1.0 +[May 1, 2026 - 01:18:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 2.0784 iter count: 1.0 +[May 1, 2026 - 01:18:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.4592 iter count: 1.0 +[May 1, 2026 - 01:18:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 1037138.3750iter count: 1.0 +[May 1, 2026 - 01:18:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 63.8699iter count: 1.0 +[May 1, 2026 - 01:18:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 22262178.0000iter count: 1.0 +[May 1, 2026 - 01:18:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6573iter count: 1.0 +[May 1, 2026 - 01:18:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.7678 iter count: 1.0 +[May 1, 2026 - 01:18:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 13844.5957iter count: 1.0 +[May 1, 2026 - 01:18:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.1944 iter count: 1.0 +[May 1, 2026 - 01:18:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 32495.5605iter count: 1.0 +[May 1, 2026 - 01:18:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.3454 iter count: 1.0 +[May 1, 2026 - 01:18:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 804.9105iter count: 1.0 +[May 1, 2026 - 01:18:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.8535 iter count: 1.0 +[May 1, 2026 - 01:18:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 897190.5625iter count: 1.0 +[May 1, 2026 - 01:18:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[May 1, 2026 - 01:18:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:18:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 01:19:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449514 41.607780 41.320610 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.432129 67.509766 67.406250 +util 93.500000 98.000000 88.000000 +[May 1, 2026 - 01:19:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 156 : avg iteration time 50.59 seconds +[May 1, 2026 - 01:19:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 156 : data loading time 0.00 +[May 1, 2026 - 01:19:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 156 : avg forward pass time 14.37 +[May 1, 2026 - 01:19:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 156 : backward pass time 10.14 +[May 1, 2026 - 01:19:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 156 : optimizer step time 1.56 +[May 1, 2026 - 01:19:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 156-------------------- +[May 1, 2026 - 01:19:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 01:19:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 01:19:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:19:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 01:20:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449822 41.609009 41.320606 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.432129 67.568359 67.386719 +util 96.500000 100.000000 92.000000 +[May 1, 2026 - 01:20:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 157 : avg iteration time 46.25 seconds +[May 1, 2026 - 01:20:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 157 : data loading time 0.00 +[May 1, 2026 - 01:20:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 157 : avg forward pass time 12.21 +[May 1, 2026 - 01:20:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 157 : backward pass time 10.14 +[May 1, 2026 - 01:20:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 157 : optimizer step time 1.56 +[May 1, 2026 - 01:20:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 157-------------------- +[May 1, 2026 - 01:20:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 01:20:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 01:20:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:20:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 01:20:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449906 41.609234 41.320690 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.446777 67.568359 67.406250 +util 94.750000 99.000000 91.000000 +[May 1, 2026 - 01:20:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 158 : avg iteration time 46.34 seconds +[May 1, 2026 - 01:20:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 158 : data loading time 0.00 +[May 1, 2026 - 01:20:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 158 : avg forward pass time 12.24 +[May 1, 2026 - 01:20:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 158 : backward pass time 10.13 +[May 1, 2026 - 01:20:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 158 : optimizer step time 1.58 +[May 1, 2026 - 01:20:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 158-------------------- +[May 1, 2026 - 01:20:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 1, 2026 - 01:20:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 1, 2026 - 01:20:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:20:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 01:21:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449909 41.609257 41.320717 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.466309 67.646484 67.406250 +util 97.000000 99.000000 93.000000 +[May 1, 2026 - 01:21:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 159 : avg iteration time 46.27 seconds +[May 1, 2026 - 01:21:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 159 : data loading time 0.00 +[May 1, 2026 - 01:21:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 159 : avg forward pass time 12.20 +[May 1, 2026 - 01:21:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 159 : backward pass time 10.15 +[May 1, 2026 - 01:21:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 159 : optimizer step time 1.57 +[May 1, 2026 - 01:21:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 159-------------------- +[May 1, 2026 - 01:21:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0086 iter count: 1.0 +[May 1, 2026 - 01:21:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 1, 2026 - 01:21:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:21:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 01:24:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449407 41.607212 41.320717 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.060547 136.095703 136.048828 +util 62.250000 67.000000 57.000000 +[May 1, 2026 - 01:24:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 160 : avg iteration time 168.76 seconds +[May 1, 2026 - 01:24:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 160 : data loading time 0.00 +[May 1, 2026 - 01:24:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 160 : avg forward pass time 75.76 +[May 1, 2026 - 01:24:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 160 : backward pass time 8.46 +[May 1, 2026 - 01:24:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 160 : optimizer step time 0.39 +[May 1, 2026 - 01:24:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 160-------------------- +[May 1, 2026 - 01:24:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 112421.3613 iter count: 1.0 +[May 1, 2026 - 01:24:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6859 iter count: 1.0 +[May 1, 2026 - 01:24:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 112421.3613iter count: 1.0 +[May 1, 2026 - 01:24:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:24:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.4091iter count: 1.0 +[May 1, 2026 - 01:24:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.6390 iter count: 1.0 +[May 1, 2026 - 01:24:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.1814 iter count: 1.0 +[May 1, 2026 - 01:24:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 166753.3125iter count: 1.0 +[May 1, 2026 - 01:24:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 3.5897 iter count: 1.0 +[May 1, 2026 - 01:24:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 1728608.5000iter count: 1.0 +[May 1, 2026 - 01:24:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6859iter count: 1.0 +[May 1, 2026 - 01:24:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.9010 iter count: 1.0 +[May 1, 2026 - 01:24:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 133523.3281iter count: 1.0 +[May 1, 2026 - 01:24:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.9259 iter count: 1.0 +[May 1, 2026 - 01:24:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 18993.6875iter count: 1.0 +[May 1, 2026 - 01:24:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.1412 iter count: 1.0 +[May 1, 2026 - 01:24:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 535.0765iter count: 1.0 +[May 1, 2026 - 01:24:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.9165 iter count: 1.0 +[May 1, 2026 - 01:24:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 2522.3416iter count: 1.0 +[May 1, 2026 - 01:24:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 1, 2026 - 01:24:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:24:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 01:25:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449414 41.607204 41.320717 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.666504 68.037109 67.542969 +util 97.500000 100.000000 93.000000 +[May 1, 2026 - 01:25:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 161 : avg iteration time 45.05 seconds +[May 1, 2026 - 01:25:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 161 : data loading time 0.00 +[May 1, 2026 - 01:25:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 161 : avg forward pass time 11.61 +[May 1, 2026 - 01:25:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 161 : backward pass time 10.13 +[May 1, 2026 - 01:25:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 161 : optimizer step time 1.58 +[May 1, 2026 - 01:25:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 161-------------------- +[May 1, 2026 - 01:25:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0084 iter count: 1.0 +[May 1, 2026 - 01:25:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 1, 2026 - 01:25:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:25:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 01:26:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449897 41.609154 41.320717 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.632324 68.017578 67.503906 +util 96.500000 100.000000 92.000000 +[May 1, 2026 - 01:26:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 162 : avg iteration time 46.39 seconds +[May 1, 2026 - 01:26:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 162 : data loading time 0.00 +[May 1, 2026 - 01:26:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 162 : avg forward pass time 12.27 +[May 1, 2026 - 01:26:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 162 : backward pass time 10.12 +[May 1, 2026 - 01:26:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 162 : optimizer step time 1.58 +[May 1, 2026 - 01:26:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 162-------------------- +[May 1, 2026 - 01:26:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0085 iter count: 1.0 +[May 1, 2026 - 01:26:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[May 1, 2026 - 01:26:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:26:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 01:26:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449979 41.609482 41.320717 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.627441 68.056641 67.484375 +util 98.500000 100.000000 95.000000 +[May 1, 2026 - 01:26:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 163 : avg iteration time 46.23 seconds +[May 1, 2026 - 01:26:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 163 : data loading time 0.00 +[May 1, 2026 - 01:26:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 163 : avg forward pass time 12.19 +[May 1, 2026 - 01:26:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 163 : backward pass time 10.13 +[May 1, 2026 - 01:26:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 163 : optimizer step time 1.58 +[May 1, 2026 - 01:26:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 163-------------------- +[May 1, 2026 - 01:26:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 1, 2026 - 01:26:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 1, 2026 - 01:26:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:26:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 01:27:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450043 41.609734 41.320717 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.710449 68.037109 67.601562 +util 93.500000 99.000000 87.000000 +[May 1, 2026 - 01:27:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 164 : avg iteration time 46.24 seconds +[May 1, 2026 - 01:27:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 164 : data loading time 0.00 +[May 1, 2026 - 01:27:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 164 : avg forward pass time 12.20 +[May 1, 2026 - 01:27:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 164 : backward pass time 10.13 +[May 1, 2026 - 01:27:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 164 : optimizer step time 1.58 +[May 1, 2026 - 01:27:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 164-------------------- +[May 1, 2026 - 01:27:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0082 iter count: 1.0 +[May 1, 2026 - 01:27:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[May 1, 2026 - 01:27:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:27:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 01:30:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449388 41.606968 41.320831 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.042480 136.076172 136.031250 +util 65.500000 69.000000 61.000000 +[May 1, 2026 - 01:30:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 165 : avg iteration time 165.74 seconds +[May 1, 2026 - 01:30:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 165 : data loading time 0.00 +[May 1, 2026 - 01:30:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 165 : avg forward pass time 74.25 +[May 1, 2026 - 01:30:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 165 : backward pass time 8.45 +[May 1, 2026 - 01:30:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 165 : optimizer step time 0.39 +[May 1, 2026 - 01:30:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 165-------------------- +[May 1, 2026 - 01:30:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 213809.3184 iter count: 1.0 +[May 1, 2026 - 01:30:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6882 iter count: 1.0 +[May 1, 2026 - 01:30:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 213809.3184iter count: 1.0 +[May 1, 2026 - 01:30:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:30:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 5.1352iter count: 1.0 +[May 1, 2026 - 01:30:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 2.2916 iter count: 1.0 +[May 1, 2026 - 01:30:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.6861 iter count: 1.0 +[May 1, 2026 - 01:30:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 285136.0938iter count: 1.0 +[May 1, 2026 - 01:30:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 97.8367iter count: 1.0 +[May 1, 2026 - 01:30:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 4743550.0000iter count: 1.0 +[May 1, 2026 - 01:30:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6882iter count: 1.0 +[May 1, 2026 - 01:30:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.3510 iter count: 1.0 +[May 1, 2026 - 01:30:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 44448.9648iter count: 1.0 +[May 1, 2026 - 01:30:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.2742 iter count: 1.0 +[May 1, 2026 - 01:30:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 38112.8750iter count: 1.0 +[May 1, 2026 - 01:30:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.4786 iter count: 1.0 +[May 1, 2026 - 01:30:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 1050.7627iter count: 1.0 +[May 1, 2026 - 01:30:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 2.5045 iter count: 1.0 +[May 1, 2026 - 01:30:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 149.7513iter count: 1.0 +[May 1, 2026 - 01:30:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 1, 2026 - 01:30:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:30:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 01:31:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449446 41.607174 41.320831 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.329590 67.451172 67.289062 +util 95.250000 99.000000 87.000000 +[May 1, 2026 - 01:31:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 166 : avg iteration time 50.45 seconds +[May 1, 2026 - 01:31:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 166 : data loading time 0.00 +[May 1, 2026 - 01:31:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 166 : avg forward pass time 14.30 +[May 1, 2026 - 01:31:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 166 : backward pass time 10.14 +[May 1, 2026 - 01:31:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 166 : optimizer step time 1.57 +[May 1, 2026 - 01:31:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 166-------------------- +[May 1, 2026 - 01:31:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 01:31:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 01:31:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:31:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 01:31:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449752 41.608421 41.320831 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.480957 67.484375 67.470703 +util 97.750000 100.000000 94.000000 +[May 1, 2026 - 01:31:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 167 : avg iteration time 48.97 seconds +[May 1, 2026 - 01:31:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 167 : data loading time 0.00 +[May 1, 2026 - 01:31:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 167 : avg forward pass time 13.57 +[May 1, 2026 - 01:31:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 167 : backward pass time 10.11 +[May 1, 2026 - 01:31:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 167 : optimizer step time 1.57 +[May 1, 2026 - 01:31:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 167-------------------- +[May 1, 2026 - 01:31:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 1, 2026 - 01:31:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 1, 2026 - 01:31:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:31:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 01:32:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449752 41.608398 41.320831 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.520020 67.626953 67.484375 +util 93.500000 100.000000 89.000000 +[May 1, 2026 - 01:32:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 168 : avg iteration time 46.30 seconds +[May 1, 2026 - 01:32:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 168 : data loading time 0.00 +[May 1, 2026 - 01:32:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 168 : avg forward pass time 12.21 +[May 1, 2026 - 01:32:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 168 : backward pass time 10.16 +[May 1, 2026 - 01:32:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 168 : optimizer step time 1.57 +[May 1, 2026 - 01:32:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 168-------------------- +[May 1, 2026 - 01:32:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0082 iter count: 1.0 +[May 1, 2026 - 01:32:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[May 1, 2026 - 01:32:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:32:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 01:33:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449744 41.608379 41.320831 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.358887 67.626953 67.269531 +util 92.500000 99.000000 88.000000 +[May 1, 2026 - 01:33:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 169 : avg iteration time 46.39 seconds +[May 1, 2026 - 01:33:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 169 : data loading time 0.00 +[May 1, 2026 - 01:33:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 169 : avg forward pass time 12.28 +[May 1, 2026 - 01:33:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 169 : backward pass time 10.13 +[May 1, 2026 - 01:33:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 169 : optimizer step time 1.57 +[May 1, 2026 - 01:33:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 169-------------------- +[May 1, 2026 - 01:33:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0077 iter count: 1.0 +[May 1, 2026 - 01:33:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0077 iter count: 1.0 +[May 1, 2026 - 01:33:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:33:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 01:36:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449670 41.607903 41.320980 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.066895 136.115234 136.050781 +util 64.250000 72.000000 57.000000 +[May 1, 2026 - 01:36:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 170 : avg iteration time 165.78 seconds +[May 1, 2026 - 01:36:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 170 : data loading time 0.00 +[May 1, 2026 - 01:36:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 170 : avg forward pass time 74.29 +[May 1, 2026 - 01:36:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 170 : backward pass time 8.42 +[May 1, 2026 - 01:36:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 170 : optimizer step time 0.39 +[May 1, 2026 - 01:36:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 170-------------------- +[May 1, 2026 - 01:36:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 1818121.0103 iter count: 1.0 +[May 1, 2026 - 01:36:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6848 iter count: 1.0 +[May 1, 2026 - 01:36:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 1818121.0103iter count: 1.0 +[May 1, 2026 - 01:36:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:36:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.2230iter count: 1.0 +[May 1, 2026 - 01:36:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.7952 iter count: 1.0 +[May 1, 2026 - 01:36:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.9715 iter count: 1.0 +[May 1, 2026 - 01:36:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 2459562.7500iter count: 1.0 +[May 1, 2026 - 01:36:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 4.9054 iter count: 1.0 +[May 1, 2026 - 01:36:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 62025260.0000iter count: 1.0 +[May 1, 2026 - 01:36:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6848iter count: 1.0 +[May 1, 2026 - 01:36:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 8.9715 iter count: 1.0 +[May 1, 2026 - 01:36:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 62025260.0000iter count: 1.0 +[May 1, 2026 - 01:36:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 6.0016 iter count: 1.0 +[May 1, 2026 - 01:36:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 163290.0625iter count: 1.0 +[May 1, 2026 - 01:36:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.4220 iter count: 1.0 +[May 1, 2026 - 01:36:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 6932.0586iter count: 1.0 +[May 1, 2026 - 01:36:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.8463 iter count: 1.0 +[May 1, 2026 - 01:36:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 2192.0754iter count: 1.0 +[May 1, 2026 - 01:36:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 1, 2026 - 01:36:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:36:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 01:37:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449665 41.607880 41.320980 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.524902 67.529297 67.523438 +util 93.250000 99.000000 89.000000 +[May 1, 2026 - 01:37:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 171 : avg iteration time 47.83 seconds +[May 1, 2026 - 01:37:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 171 : data loading time 0.00 +[May 1, 2026 - 01:37:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 171 : avg forward pass time 12.99 +[May 1, 2026 - 01:37:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 171 : backward pass time 10.14 +[May 1, 2026 - 01:37:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 171 : optimizer step time 1.59 +[May 1, 2026 - 01:37:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 171-------------------- +[May 1, 2026 - 01:37:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0076 iter count: 1.0 +[May 1, 2026 - 01:37:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0076 iter count: 1.0 +[May 1, 2026 - 01:37:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:37:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 01:37:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449862 41.608604 41.320980 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.520020 67.523438 67.509766 +util 93.250000 99.000000 89.000000 +[May 1, 2026 - 01:37:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 172 : avg iteration time 46.20 seconds +[May 1, 2026 - 01:37:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 172 : data loading time 0.00 +[May 1, 2026 - 01:37:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 172 : avg forward pass time 12.17 +[May 1, 2026 - 01:37:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 172 : backward pass time 10.13 +[May 1, 2026 - 01:37:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 172 : optimizer step time 1.57 +[May 1, 2026 - 01:37:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 172-------------------- +[May 1, 2026 - 01:37:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 1, 2026 - 01:37:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 1, 2026 - 01:37:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:37:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 01:38:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449972 41.609085 41.321003 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.520020 67.523438 67.509766 +util 95.500000 99.000000 91.000000 +[May 1, 2026 - 01:38:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 173 : avg iteration time 46.27 seconds +[May 1, 2026 - 01:38:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 173 : data loading time 0.00 +[May 1, 2026 - 01:38:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 173 : avg forward pass time 12.19 +[May 1, 2026 - 01:38:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 173 : backward pass time 10.16 +[May 1, 2026 - 01:38:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 173 : optimizer step time 1.56 +[May 1, 2026 - 01:38:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 173-------------------- +[May 1, 2026 - 01:38:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 01:38:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 01:38:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:38:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 01:39:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449873 41.608658 41.320980 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.539551 67.542969 67.529297 +util 97.250000 100.000000 93.000000 +[May 1, 2026 - 01:39:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 174 : avg iteration time 46.42 seconds +[May 1, 2026 - 01:39:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 174 : data loading time 0.00 +[May 1, 2026 - 01:39:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 174 : avg forward pass time 12.27 +[May 1, 2026 - 01:39:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 174 : backward pass time 10.14 +[May 1, 2026 - 01:39:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 174 : optimizer step time 1.56 +[May 1, 2026 - 01:39:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 174-------------------- +[May 1, 2026 - 01:39:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 1, 2026 - 01:39:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 1, 2026 - 01:39:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:39:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 01:42:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449715 41.607918 41.320988 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.018066 136.154297 135.972656 +util 69.000000 73.000000 65.000000 +[May 1, 2026 - 01:42:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 175 : avg iteration time 163.24 seconds +[May 1, 2026 - 01:42:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 175 : data loading time 0.00 +[May 1, 2026 - 01:42:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 175 : avg forward pass time 73.00 +[May 1, 2026 - 01:42:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 175 : backward pass time 8.42 +[May 1, 2026 - 01:42:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 175 : optimizer step time 0.40 +[May 1, 2026 - 01:42:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 175-------------------- +[May 1, 2026 - 01:42:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 878172.5361 iter count: 1.0 +[May 1, 2026 - 01:42:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6658 iter count: 1.0 +[May 1, 2026 - 01:42:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 878172.5361iter count: 1.0 +[May 1, 2026 - 01:42:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:42:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.7060iter count: 1.0 +[May 1, 2026 - 01:42:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.3665 iter count: 1.0 +[May 1, 2026 - 01:42:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.5475 iter count: 1.0 +[May 1, 2026 - 01:42:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 1270418.2500iter count: 1.0 +[May 1, 2026 - 01:42:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 2.0815 iter count: 1.0 +[May 1, 2026 - 01:42:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 26564268.0000iter count: 1.0 +[May 1, 2026 - 01:42:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6658iter count: 1.0 +[May 1, 2026 - 01:42:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.8065 iter count: 1.0 +[May 1, 2026 - 01:42:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 14957.2480iter count: 1.0 +[May 1, 2026 - 01:42:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.0335 iter count: 1.0 +[May 1, 2026 - 01:42:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 3187.4746iter count: 1.0 +[May 1, 2026 - 01:42:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 6.0177 iter count: 1.0 +[May 1, 2026 - 01:42:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 168623.9219iter count: 1.0 +[May 1, 2026 - 01:42:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 2.6045 iter count: 1.0 +[May 1, 2026 - 01:42:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 182.9176iter count: 1.0 +[May 1, 2026 - 01:42:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 1, 2026 - 01:42:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:42:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 01:42:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449709 41.607918 41.320984 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.441895 67.490234 67.425781 +util 92.500000 99.000000 88.000000 +[May 1, 2026 - 01:42:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 176 : avg iteration time 47.87 seconds +[May 1, 2026 - 01:42:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 176 : data loading time 0.00 +[May 1, 2026 - 01:42:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 176 : avg forward pass time 13.00 +[May 1, 2026 - 01:42:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 176 : backward pass time 10.15 +[May 1, 2026 - 01:42:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 176 : optimizer step time 1.57 +[May 1, 2026 - 01:42:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 176-------------------- +[May 1, 2026 - 01:42:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 1, 2026 - 01:42:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 1, 2026 - 01:42:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:42:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 01:43:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449899 41.608631 41.320984 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.554199 67.582031 67.470703 +util 94.250000 100.000000 90.000000 +[May 1, 2026 - 01:43:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 177 : avg iteration time 46.22 seconds +[May 1, 2026 - 01:43:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 177 : data loading time 0.00 +[May 1, 2026 - 01:43:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 177 : avg forward pass time 12.20 +[May 1, 2026 - 01:43:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 177 : backward pass time 10.11 +[May 1, 2026 - 01:43:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 177 : optimizer step time 1.56 +[May 1, 2026 - 01:43:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 177-------------------- +[May 1, 2026 - 01:43:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0104 iter count: 1.0 +[May 1, 2026 - 01:43:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 1, 2026 - 01:43:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:43:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 01:44:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449894 41.608631 41.321003 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.485840 67.490234 67.484375 +util 98.250000 100.000000 95.000000 +[May 1, 2026 - 01:44:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 178 : avg iteration time 43.71 seconds +[May 1, 2026 - 01:44:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 178 : data loading time 0.00 +[May 1, 2026 - 01:44:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 178 : avg forward pass time 10.91 +[May 1, 2026 - 01:44:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 178 : backward pass time 10.15 +[May 1, 2026 - 01:44:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 178 : optimizer step time 1.57 +[May 1, 2026 - 01:44:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 178-------------------- +[May 1, 2026 - 01:44:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 01:44:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 01:44:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:44:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 01:45:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449899 41.608650 41.321003 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.500488 67.548828 67.484375 +util 95.250000 99.000000 91.000000 +[May 1, 2026 - 01:45:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 179 : avg iteration time 48.95 seconds +[May 1, 2026 - 01:45:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 179 : data loading time 0.00 +[May 1, 2026 - 01:45:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 179 : avg forward pass time 13.54 +[May 1, 2026 - 01:45:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 179 : backward pass time 10.14 +[May 1, 2026 - 01:45:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 179 : optimizer step time 1.58 +[May 1, 2026 - 01:45:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 179-------------------- +[May 1, 2026 - 01:45:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 1, 2026 - 01:45:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 1, 2026 - 01:45:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:45:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 01:48:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449932 41.608143 41.321228 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 135.983887 136.076172 135.953125 +util 69.250000 74.000000 65.000000 +[May 1, 2026 - 01:48:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 180 : avg iteration time 168.46 seconds +[May 1, 2026 - 01:48:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 180 : data loading time 0.00 +[May 1, 2026 - 01:48:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 180 : avg forward pass time 75.63 +[May 1, 2026 - 01:48:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 180 : backward pass time 8.43 +[May 1, 2026 - 01:48:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 180 : optimizer step time 0.39 +[May 1, 2026 - 01:48:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 180-------------------- +[May 1, 2026 - 01:48:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 106106.3149 iter count: 1.0 +[May 1, 2026 - 01:48:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7037 iter count: 1.0 +[May 1, 2026 - 01:48:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 106106.3149iter count: 1.0 +[May 1, 2026 - 01:48:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:48:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.4441iter count: 1.0 +[May 1, 2026 - 01:48:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.7850 iter count: 1.0 +[May 1, 2026 - 01:48:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.2245 iter count: 1.0 +[May 1, 2026 - 01:48:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 177161.5469iter count: 1.0 +[May 1, 2026 - 01:48:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 4.8070 iter count: 1.0 +[May 1, 2026 - 01:48:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 1884215.8750iter count: 1.0 +[May 1, 2026 - 01:48:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7037iter count: 1.0 +[May 1, 2026 - 01:48:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.7050 iter count: 1.0 +[May 1, 2026 - 01:48:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 1652.4451iter count: 1.0 +[May 1, 2026 - 01:48:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.0719 iter count: 1.0 +[May 1, 2026 - 01:48:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 3441.6470iter count: 1.0 +[May 1, 2026 - 01:48:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 2.4417 iter count: 1.0 +[May 1, 2026 - 01:48:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 132.0822iter count: 1.0 +[May 1, 2026 - 01:48:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.1424 iter count: 1.0 +[May 1, 2026 - 01:48:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 3962.8755iter count: 1.0 +[May 1, 2026 - 01:48:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 01:48:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:48:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 01:48:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450000 41.608391 41.321251 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.544434 67.666016 67.503906 +util 96.000000 99.000000 89.000000 +[May 1, 2026 - 01:48:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 181 : avg iteration time 47.81 seconds +[May 1, 2026 - 01:48:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 181 : data loading time 0.00 +[May 1, 2026 - 01:48:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 181 : avg forward pass time 12.98 +[May 1, 2026 - 01:48:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 181 : backward pass time 10.13 +[May 1, 2026 - 01:48:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 181 : optimizer step time 1.58 +[May 1, 2026 - 01:48:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 181-------------------- +[May 1, 2026 - 01:48:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0127 iter count: 1.0 +[May 1, 2026 - 01:48:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0127 iter count: 1.0 +[May 1, 2026 - 01:48:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:48:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 01:49:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449993 41.608376 41.321262 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.495605 67.646484 67.445312 +util 98.000000 100.000000 94.000000 +[May 1, 2026 - 01:49:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 182 : avg iteration time 46.36 seconds +[May 1, 2026 - 01:49:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 182 : data loading time 0.00 +[May 1, 2026 - 01:49:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 182 : avg forward pass time 12.24 +[May 1, 2026 - 01:49:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 182 : backward pass time 10.16 +[May 1, 2026 - 01:49:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 182 : optimizer step time 1.56 +[May 1, 2026 - 01:49:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 182-------------------- +[May 1, 2026 - 01:49:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 01:49:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 01:49:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:49:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 01:50:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450113 41.608856 41.321262 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.480957 67.646484 67.425781 +util 93.500000 98.000000 89.000000 +[May 1, 2026 - 01:50:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 183 : avg iteration time 48.93 seconds +[May 1, 2026 - 01:50:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 183 : data loading time 0.00 +[May 1, 2026 - 01:50:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 183 : avg forward pass time 13.53 +[May 1, 2026 - 01:50:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 183 : backward pass time 10.15 +[May 1, 2026 - 01:50:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 183 : optimizer step time 1.57 +[May 1, 2026 - 01:50:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 183-------------------- +[May 1, 2026 - 01:50:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 1, 2026 - 01:50:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 1, 2026 - 01:50:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:50:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 01:51:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450166 41.609089 41.321239 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.510254 67.646484 67.464844 +util 92.250000 96.000000 89.000000 +[May 1, 2026 - 01:51:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 184 : avg iteration time 43.59 seconds +[May 1, 2026 - 01:51:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 184 : data loading time 0.00 +[May 1, 2026 - 01:51:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 184 : avg forward pass time 10.84 +[May 1, 2026 - 01:51:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 184 : backward pass time 10.14 +[May 1, 2026 - 01:51:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 184 : optimizer step time 1.59 +[May 1, 2026 - 01:51:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 184-------------------- +[May 1, 2026 - 01:51:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 01:51:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 01:51:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:51:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 01:53:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449986 41.608242 41.321327 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.006836 136.173828 135.951172 +util 62.500000 71.000000 57.000000 +[May 1, 2026 - 01:53:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 185 : avg iteration time 163.17 seconds +[May 1, 2026 - 01:53:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 185 : data loading time 0.00 +[May 1, 2026 - 01:53:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 185 : avg forward pass time 72.96 +[May 1, 2026 - 01:53:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 185 : backward pass time 8.42 +[May 1, 2026 - 01:53:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 185 : optimizer step time 0.40 +[May 1, 2026 - 01:53:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 185-------------------- +[May 1, 2026 - 01:53:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 270944.9982 iter count: 1.0 +[May 1, 2026 - 01:53:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6451 iter count: 1.0 +[May 1, 2026 - 01:53:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 270944.9982iter count: 1.0 +[May 1, 2026 - 01:53:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:53:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.2072iter count: 1.0 +[May 1, 2026 - 01:53:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.1722 iter count: 1.0 +[May 1, 2026 - 01:53:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.7512 iter count: 1.0 +[May 1, 2026 - 01:53:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 352264.8750iter count: 1.0 +[May 1, 2026 - 01:53:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 10.4276iter count: 1.0 +[May 1, 2026 - 01:53:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 5402734.0000iter count: 1.0 +[May 1, 2026 - 01:53:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6451iter count: 1.0 +[May 1, 2026 - 01:53:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.8821 iter count: 1.0 +[May 1, 2026 - 01:53:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 17400.7559iter count: 1.0 +[May 1, 2026 - 01:53:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.6801 iter count: 1.0 +[May 1, 2026 - 01:53:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 1572.2273iter count: 1.0 +[May 1, 2026 - 01:53:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.8593 iter count: 1.0 +[May 1, 2026 - 01:53:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 2249.9304iter count: 1.0 +[May 1, 2026 - 01:53:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 2.4637 iter count: 1.0 +[May 1, 2026 - 01:53:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 138.0105iter count: 1.0 +[May 1, 2026 - 01:53:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 01:53:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:53:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 01:54:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450006 41.608257 41.321377 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.959473 67.978516 67.953125 +util 96.000000 100.000000 88.000000 +[May 1, 2026 - 01:54:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 186 : avg iteration time 50.53 seconds +[May 1, 2026 - 01:54:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 186 : data loading time 0.00 +[May 1, 2026 - 01:54:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 186 : avg forward pass time 14.34 +[May 1, 2026 - 01:54:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 186 : backward pass time 10.14 +[May 1, 2026 - 01:54:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 186 : optimizer step time 1.57 +[May 1, 2026 - 01:54:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 186-------------------- +[May 1, 2026 - 01:54:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 1, 2026 - 01:54:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 1, 2026 - 01:54:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:54:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 01:55:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450089 41.608601 41.321377 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.441895 68.134766 67.210938 +util 95.750000 99.000000 88.000000 +[May 1, 2026 - 01:55:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 187 : avg iteration time 43.60 seconds +[May 1, 2026 - 01:55:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 187 : data loading time 0.00 +[May 1, 2026 - 01:55:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 187 : avg forward pass time 10.88 +[May 1, 2026 - 01:55:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 187 : backward pass time 10.14 +[May 1, 2026 - 01:55:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 187 : optimizer step time 1.56 +[May 1, 2026 - 01:55:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 187-------------------- +[May 1, 2026 - 01:55:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0083 iter count: 1.0 +[May 1, 2026 - 01:55:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[May 1, 2026 - 01:55:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:55:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 01:56:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450098 41.608601 41.321365 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.598145 67.998047 67.464844 +util 96.500000 99.000000 92.000000 +[May 1, 2026 - 01:56:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 188 : avg iteration time 43.56 seconds +[May 1, 2026 - 01:56:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 188 : data loading time 0.00 +[May 1, 2026 - 01:56:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 188 : avg forward pass time 10.84 +[May 1, 2026 - 01:56:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 188 : backward pass time 10.15 +[May 1, 2026 - 01:56:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 188 : optimizer step time 1.58 +[May 1, 2026 - 01:56:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 188-------------------- +[May 1, 2026 - 01:56:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 01:56:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 01:56:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:56:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 01:56:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450150 41.608841 41.321354 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.607910 67.978516 67.484375 +util 93.250000 97.000000 89.000000 +[May 1, 2026 - 01:56:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 189 : avg iteration time 49.14 seconds +[May 1, 2026 - 01:56:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 189 : data loading time 0.00 +[May 1, 2026 - 01:56:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 189 : avg forward pass time 13.64 +[May 1, 2026 - 01:56:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 189 : backward pass time 10.14 +[May 1, 2026 - 01:56:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 189 : optimizer step time 1.56 +[May 1, 2026 - 01:56:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 189-------------------- +[May 1, 2026 - 01:56:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 01:56:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 01:56:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:56:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 01:59:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449921 41.607883 41.321404 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.076660 136.154297 136.050781 +util 66.750000 71.000000 63.000000 +[May 1, 2026 - 01:59:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 190 : avg iteration time 168.46 seconds +[May 1, 2026 - 01:59:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 190 : data loading time 0.00 +[May 1, 2026 - 01:59:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 190 : avg forward pass time 75.63 +[May 1, 2026 - 01:59:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 190 : backward pass time 8.42 +[May 1, 2026 - 01:59:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 190 : optimizer step time 0.39 +[May 1, 2026 - 01:59:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 190-------------------- +[May 1, 2026 - 01:59:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 449977.6279 iter count: 1.0 +[May 1, 2026 - 01:59:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6154 iter count: 1.0 +[May 1, 2026 - 01:59:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 449977.6279iter count: 1.0 +[May 1, 2026 - 01:59:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:59:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.8460iter count: 1.0 +[May 1, 2026 - 01:59:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.3787 iter count: 1.0 +[May 1, 2026 - 01:59:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.7175 iter count: 1.0 +[May 1, 2026 - 01:59:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 719066.8125iter count: 1.0 +[May 1, 2026 - 01:59:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 15.7603iter count: 1.0 +[May 1, 2026 - 01:59:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 5050089.5000iter count: 1.0 +[May 1, 2026 - 01:59:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6154iter count: 1.0 +[May 1, 2026 - 01:59:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.9017 iter count: 1.0 +[May 1, 2026 - 01:59:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 133700.7031iter count: 1.0 +[May 1, 2026 - 01:59:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.9083 iter count: 1.0 +[May 1, 2026 - 01:59:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 2481.5271iter count: 1.0 +[May 1, 2026 - 01:59:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.0333 iter count: 1.0 +[May 1, 2026 - 01:59:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 431.1713iter count: 1.0 +[May 1, 2026 - 01:59:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.5462 iter count: 1.0 +[May 1, 2026 - 01:59:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 8886.9951iter count: 1.0 +[May 1, 2026 - 01:59:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 1, 2026 - 01:59:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 01:59:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 02:00:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450037 41.608349 41.321392 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.559570 67.726562 67.503906 +util 94.500000 99.000000 90.000000 +[May 1, 2026 - 02:00:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 191 : avg iteration time 50.56 seconds +[May 1, 2026 - 02:00:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 191 : data loading time 0.00 +[May 1, 2026 - 02:00:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 191 : avg forward pass time 14.34 +[May 1, 2026 - 02:00:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 191 : backward pass time 10.16 +[May 1, 2026 - 02:00:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 191 : optimizer step time 1.58 +[May 1, 2026 - 02:00:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 191-------------------- +[May 1, 2026 - 02:00:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 1, 2026 - 02:00:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 1, 2026 - 02:00:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:00:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 02:01:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450241 41.609089 41.321415 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.613281 67.640625 67.531250 +util 94.750000 99.000000 89.000000 +[May 1, 2026 - 02:01:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 192 : avg iteration time 43.72 seconds +[May 1, 2026 - 02:01:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 192 : data loading time 0.00 +[May 1, 2026 - 02:01:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 192 : avg forward pass time 10.91 +[May 1, 2026 - 02:01:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 192 : backward pass time 10.16 +[May 1, 2026 - 02:01:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 192 : optimizer step time 1.57 +[May 1, 2026 - 02:01:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 192-------------------- +[May 1, 2026 - 02:01:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 1, 2026 - 02:01:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 1, 2026 - 02:01:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:01:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 02:02:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450238 41.609093 41.321400 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.750000 67.777344 67.667969 +util 97.500000 100.000000 94.000000 +[May 1, 2026 - 02:02:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 193 : avg iteration time 48.93 seconds +[May 1, 2026 - 02:02:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 193 : data loading time 0.00 +[May 1, 2026 - 02:02:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 193 : avg forward pass time 13.54 +[May 1, 2026 - 02:02:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 193 : backward pass time 10.13 +[May 1, 2026 - 02:02:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 193 : optimizer step time 1.56 +[May 1, 2026 - 02:02:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 193-------------------- +[May 1, 2026 - 02:02:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0082 iter count: 1.0 +[May 1, 2026 - 02:02:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[May 1, 2026 - 02:02:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:02:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 02:03:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450296 41.609333 41.321392 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.618164 67.667969 67.601562 +util 96.000000 99.000000 92.000000 +[May 1, 2026 - 02:03:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 194 : avg iteration time 48.98 seconds +[May 1, 2026 - 02:03:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 194 : data loading time 0.00 +[May 1, 2026 - 02:03:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 194 : avg forward pass time 13.57 +[May 1, 2026 - 02:03:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 194 : backward pass time 10.14 +[May 1, 2026 - 02:03:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 194 : optimizer step time 1.55 +[May 1, 2026 - 02:03:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 194-------------------- +[May 1, 2026 - 02:03:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0083 iter count: 1.0 +[May 1, 2026 - 02:03:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[May 1, 2026 - 02:03:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:03:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 02:05:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450066 41.608406 41.321396 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.001953 136.037109 135.990234 +util 65.250000 69.000000 61.000000 +[May 1, 2026 - 02:05:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 195 : avg iteration time 160.60 seconds +[May 1, 2026 - 02:05:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 195 : data loading time 0.00 +[May 1, 2026 - 02:05:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 195 : avg forward pass time 71.70 +[May 1, 2026 - 02:05:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 195 : backward pass time 8.39 +[May 1, 2026 - 02:05:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 195 : optimizer step time 0.39 +[May 1, 2026 - 02:05:40 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 195-------------------- +[May 1, 2026 - 02:05:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 322483.0312 iter count: 1.0 +[May 1, 2026 - 02:05:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5922 iter count: 1.0 +[May 1, 2026 - 02:05:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 322483.0312iter count: 1.0 +[May 1, 2026 - 02:05:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:05:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.7572iter count: 1.0 +[May 1, 2026 - 02:05:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.7012 iter count: 1.0 +[May 1, 2026 - 02:05:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.8671 iter count: 1.0 +[May 1, 2026 - 02:05:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 562654.9375iter count: 1.0 +[May 1, 2026 - 02:05:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 4.0646 iter count: 1.0 +[May 1, 2026 - 02:05:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 6812671.5000iter count: 1.0 +[May 1, 2026 - 02:05:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5922iter count: 1.0 +[May 1, 2026 - 02:05:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.5835 iter count: 1.0 +[May 1, 2026 - 02:05:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 1295.9236iter count: 1.0 +[May 1, 2026 - 02:05:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 2.8074 iter count: 1.0 +[May 1, 2026 - 02:05:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 274.4463iter count: 1.0 +[May 1, 2026 - 02:05:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 6.5843 iter count: 1.0 +[May 1, 2026 - 02:05:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 523635.6875iter count: 1.0 +[May 1, 2026 - 02:05:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 7.8671 iter count: 1.0 +[May 1, 2026 - 02:05:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 6812671.5000iter count: 1.0 +[May 1, 2026 - 02:05:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[May 1, 2026 - 02:05:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:05:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 02:06:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450128 41.608654 41.321392 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.691406 68.136719 67.542969 +util 93.750000 100.000000 87.000000 +[May 1, 2026 - 02:06:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 196 : avg iteration time 50.67 seconds +[May 1, 2026 - 02:06:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 196 : data loading time 0.00 +[May 1, 2026 - 02:06:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 196 : avg forward pass time 14.40 +[May 1, 2026 - 02:06:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 196 : backward pass time 10.15 +[May 1, 2026 - 02:06:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 196 : optimizer step time 1.57 +[May 1, 2026 - 02:06:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 196-------------------- +[May 1, 2026 - 02:06:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 02:06:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 02:06:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:06:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 02:07:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450375 41.609631 41.321404 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.681641 68.507812 67.406250 +util 96.750000 99.000000 93.000000 +[May 1, 2026 - 02:07:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 197 : avg iteration time 48.94 seconds +[May 1, 2026 - 02:07:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 197 : data loading time 0.00 +[May 1, 2026 - 02:07:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 197 : avg forward pass time 13.54 +[May 1, 2026 - 02:07:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 197 : backward pass time 10.13 +[May 1, 2026 - 02:07:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 197 : optimizer step time 1.58 +[May 1, 2026 - 02:07:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 197-------------------- +[May 1, 2026 - 02:07:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0100 iter count: 1.0 +[May 1, 2026 - 02:07:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 1, 2026 - 02:07:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:07:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 02:08:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450434 41.609867 41.321404 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.515625 67.902344 67.386719 +util 94.250000 100.000000 90.000000 +[May 1, 2026 - 02:08:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 198 : avg iteration time 46.33 seconds +[May 1, 2026 - 02:08:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 198 : data loading time 0.00 +[May 1, 2026 - 02:08:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 198 : avg forward pass time 12.24 +[May 1, 2026 - 02:08:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 198 : backward pass time 10.15 +[May 1, 2026 - 02:08:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 198 : optimizer step time 1.57 +[May 1, 2026 - 02:08:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 198-------------------- +[May 1, 2026 - 02:08:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 1, 2026 - 02:08:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 1, 2026 - 02:08:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:08:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 02:08:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450265 41.609188 41.321404 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.510742 68.000000 67.347656 +util 92.250000 99.000000 85.000000 +[May 1, 2026 - 02:08:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 199 : avg iteration time 46.33 seconds +[May 1, 2026 - 02:08:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 199 : data loading time 0.00 +[May 1, 2026 - 02:08:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 199 : avg forward pass time 12.23 +[May 1, 2026 - 02:08:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 199 : backward pass time 10.16 +[May 1, 2026 - 02:08:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 199 : optimizer step time 1.58 +[May 1, 2026 - 02:08:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 199-------------------- +[May 1, 2026 - 02:08:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 02:08:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 02:08:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:08:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 02:11:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450167 41.608646 41.321522 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.041016 136.076172 136.029297 +util 63.750000 69.000000 58.000000 +[May 1, 2026 - 02:11:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 200 : avg iteration time 168.40 seconds +[May 1, 2026 - 02:11:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 200 : data loading time 0.00 +[May 1, 2026 - 02:11:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 200 : avg forward pass time 75.58 +[May 1, 2026 - 02:11:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 200 : backward pass time 8.41 +[May 1, 2026 - 02:11:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 200 : optimizer step time 0.39 +[May 1, 2026 - 02:11:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 200-------------------- +[May 1, 2026 - 02:11:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 98348.6758 iter count: 1.0 +[May 1, 2026 - 02:11:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6499 iter count: 1.0 +[May 1, 2026 - 02:11:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 98348.6758iter count: 1.0 +[May 1, 2026 - 02:11:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:11:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.6709iter count: 1.0 +[May 1, 2026 - 02:11:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.5467 iter count: 1.0 +[May 1, 2026 - 02:11:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.2281 iter count: 1.0 +[May 1, 2026 - 02:11:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 163847.6875iter count: 1.0 +[May 1, 2026 - 02:11:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 2.9846 iter count: 1.0 +[May 1, 2026 - 02:11:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 1897739.2500iter count: 1.0 +[May 1, 2026 - 02:11:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6499iter count: 1.0 +[May 1, 2026 - 02:11:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.6673 iter count: 1.0 +[May 1, 2026 - 02:11:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 1532.5176iter count: 1.0 +[May 1, 2026 - 02:11:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.1740 iter count: 1.0 +[May 1, 2026 - 02:11:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 4221.8882iter count: 1.0 +[May 1, 2026 - 02:11:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.9869 iter count: 1.0 +[May 1, 2026 - 02:11:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 2903.6958iter count: 1.0 +[May 1, 2026 - 02:11:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.4910 iter count: 1.0 +[May 1, 2026 - 02:11:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 1077.0317iter count: 1.0 +[May 1, 2026 - 02:11:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 02:11:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 1, 2026 - 02:11:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 14.89 seconds +[May 1, 2026 - 02:11:56 | INFO | fastgen.trainer:save_checkpoint:266 ] Saving checkpoint iteration 200 +[May 1, 2026 - 02:11:56 | INFO | fastgen.utils.checkpointer:save:311 ] Saving FSDP model to prefix /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000200 +[May 1, 2026 - 02:12:50 | SUCCESS | fastgen.utils.checkpointer:save:362 ] Model saved at iteration 200 +[May 1, 2026 - 02:12:51 | INFO | fastgen.trainer:validate:358 ] Validation iteration 200 +[May 1, 2026 - 02:12:51 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 02:12:53 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 2.2s +[May 1, 2026 - 02:12:56 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 0: sync_c=6.515 +[May 1, 2026 - 02:12:56 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 02:12:58 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 02:13:01 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 1: sync_c=5.638 +[May 1, 2026 - 02:13:02 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 02:13:03 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 02:13:06 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 2: sync_c=4.795 +[May 1, 2026 - 02:13:07 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 02:13:08 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 02:13:11 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 3: sync_c=4.677 +[May 1, 2026 - 02:13:12 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 02:13:13 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 02:13:16 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 4: sync_c=4.476 +[May 1, 2026 - 02:13:17 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 02:13:19 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 02:13:22 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 5: sync_c=7.131 +[May 1, 2026 - 02:13:22 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 02:13:24 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 02:13:27 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 6: sync_c=6.599 +[May 1, 2026 - 02:13:27 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 02:13:29 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 02:13:32 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 7: sync_c=4.845 +[May 1, 2026 - 02:13:32 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 02:13:34 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 02:13:37 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 8: sync_c=6.042 +[May 1, 2026 - 02:13:37 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 02:13:39 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 02:13:42 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 9: sync_c=2.566 +[May 1, 2026 - 02:13:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging val0 stats at iteration 200-------------------- +[May 1, 2026 - 02:13:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0000 iter count: 10.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 1, 2026 - 02:13:58 | INFO | fastgen.callbacks.wandb:on_validation_end:763 ] Logged 10 val videos at iteration 200 +[May 1, 2026 - 02:13:58 | INFO | fastgen.callbacks.wandb:on_validation_end:769 ] [SyncEval] val0 mean sync_c: 5.328 (n=10) +[May 1, 2026 - 02:14:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.451619 41.615704 41.321201 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.027832 68.662109 67.816406 +util 93.750000 99.000000 89.000000 +[May 1, 2026 - 02:14:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 201 : avg iteration time 183.07 seconds +[May 1, 2026 - 02:14:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 201 : data loading time 0.00 +[May 1, 2026 - 02:14:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 201 : avg forward pass time 11.90 +[May 1, 2026 - 02:14:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 201 : backward pass time 10.13 +[May 1, 2026 - 02:14:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 201 : optimizer step time 1.57 +[May 1, 2026 - 02:14:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 201-------------------- +[May 1, 2026 - 02:14:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 02:14:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 02:14:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:14:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 02:15:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.451157 41.613842 41.321201 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.003418 68.623047 67.796875 +util 95.750000 100.000000 90.000000 +[May 1, 2026 - 02:15:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 202 : avg iteration time 49.17 seconds +[May 1, 2026 - 02:15:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 202 : data loading time 0.00 +[May 1, 2026 - 02:15:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 202 : avg forward pass time 13.62 +[May 1, 2026 - 02:15:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 202 : backward pass time 10.18 +[May 1, 2026 - 02:15:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 202 : optimizer step time 1.56 +[May 1, 2026 - 02:15:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 202-------------------- +[May 1, 2026 - 02:15:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 02:15:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 02:15:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:15:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 02:16:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.451169 41.613842 41.321228 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.057129 68.720703 67.835938 +util 97.250000 100.000000 93.000000 +[May 1, 2026 - 02:16:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 203 : avg iteration time 46.52 seconds +[May 1, 2026 - 02:16:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 203 : data loading time 0.00 +[May 1, 2026 - 02:16:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 203 : avg forward pass time 12.28 +[May 1, 2026 - 02:16:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 203 : backward pass time 10.16 +[May 1, 2026 - 02:16:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 203 : optimizer step time 1.60 +[May 1, 2026 - 02:16:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 203-------------------- +[May 1, 2026 - 02:16:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 02:16:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 02:16:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:16:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 02:17:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450520 41.611275 41.321228 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.027832 68.662109 67.816406 +util 95.750000 100.000000 88.000000 +[May 1, 2026 - 02:17:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 204 : avg iteration time 43.62 seconds +[May 1, 2026 - 02:17:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 204 : data loading time 0.00 +[May 1, 2026 - 02:17:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 204 : avg forward pass time 10.87 +[May 1, 2026 - 02:17:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 204 : backward pass time 10.14 +[May 1, 2026 - 02:17:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 204 : optimizer step time 1.58 +[May 1, 2026 - 02:17:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 204-------------------- +[May 1, 2026 - 02:17:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0080 iter count: 1.0 +[May 1, 2026 - 02:17:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0080 iter count: 1.0 +[May 1, 2026 - 02:17:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:17:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 02:19:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450407 41.609329 41.321682 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.065430 136.068359 136.056641 +util 65.000000 70.000000 61.000000 +[May 1, 2026 - 02:19:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 205 : avg iteration time 166.19 seconds +[May 1, 2026 - 02:19:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 205 : data loading time 0.00 +[May 1, 2026 - 02:19:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 205 : avg forward pass time 74.45 +[May 1, 2026 - 02:19:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 205 : backward pass time 8.45 +[May 1, 2026 - 02:19:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 205 : optimizer step time 0.43 +[May 1, 2026 - 02:19:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 205-------------------- +[May 1, 2026 - 02:19:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 177894.6162 iter count: 1.0 +[May 1, 2026 - 02:19:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6239 iter count: 1.0 +[May 1, 2026 - 02:19:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 177894.6162iter count: 1.0 +[May 1, 2026 - 02:19:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:19:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.1656iter count: 1.0 +[May 1, 2026 - 02:19:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.5106 iter count: 1.0 +[May 1, 2026 - 02:19:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.5168 iter count: 1.0 +[May 1, 2026 - 02:19:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 257808.0156iter count: 1.0 +[May 1, 2026 - 02:19:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 2.7765 iter count: 1.0 +[May 1, 2026 - 02:19:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 3381009.0000iter count: 1.0 +[May 1, 2026 - 02:19:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6239iter count: 1.0 +[May 1, 2026 - 02:19:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 6.4729 iter count: 1.0 +[May 1, 2026 - 02:19:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 419053.1875iter count: 1.0 +[May 1, 2026 - 02:19:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 2.0257 iter count: 1.0 +[May 1, 2026 - 02:19:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 57.4782 iter count: 1.0 +[May 1, 2026 - 02:19:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.8937 iter count: 1.0 +[May 1, 2026 - 02:19:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 17807.0840iter count: 1.0 +[May 1, 2026 - 02:19:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.2510 iter count: 1.0 +[May 1, 2026 - 02:19:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 36384.6953iter count: 1.0 +[May 1, 2026 - 02:19:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 1, 2026 - 02:19:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:19:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 02:20:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450397 41.609344 41.321697 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.662109 67.679688 67.609375 +util 93.250000 100.000000 88.000000 +[May 1, 2026 - 02:20:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 206 : avg iteration time 47.83 seconds +[May 1, 2026 - 02:20:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 206 : data loading time 0.00 +[May 1, 2026 - 02:20:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 206 : avg forward pass time 12.97 +[May 1, 2026 - 02:20:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 206 : backward pass time 10.15 +[May 1, 2026 - 02:20:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 206 : optimizer step time 1.59 +[May 1, 2026 - 02:20:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 206-------------------- +[May 1, 2026 - 02:20:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 02:20:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 02:20:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:20:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 02:21:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450781 41.610878 41.321682 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.129883 67.472656 67.015625 +util 93.000000 97.000000 89.000000 +[May 1, 2026 - 02:21:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 207 : avg iteration time 46.29 seconds +[May 1, 2026 - 02:21:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 207 : data loading time 0.00 +[May 1, 2026 - 02:21:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 207 : avg forward pass time 12.22 +[May 1, 2026 - 02:21:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 207 : backward pass time 10.14 +[May 1, 2026 - 02:21:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 207 : optimizer step time 1.58 +[May 1, 2026 - 02:21:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 207-------------------- +[May 1, 2026 - 02:21:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 02:21:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 02:21:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:21:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 02:22:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450830 41.611065 41.321682 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.515625 67.550781 67.503906 +util 94.250000 100.000000 89.000000 +[May 1, 2026 - 02:22:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 208 : avg iteration time 46.38 seconds +[May 1, 2026 - 02:22:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 208 : data loading time 0.00 +[May 1, 2026 - 02:22:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 208 : avg forward pass time 12.26 +[May 1, 2026 - 02:22:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 208 : backward pass time 10.16 +[May 1, 2026 - 02:22:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 208 : optimizer step time 1.57 +[May 1, 2026 - 02:22:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 208-------------------- +[May 1, 2026 - 02:22:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 1, 2026 - 02:22:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 1, 2026 - 02:22:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:22:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 02:22:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450611 41.610188 41.321686 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.115234 67.472656 66.996094 +util 95.000000 100.000000 89.000000 +[May 1, 2026 - 02:22:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 209 : avg iteration time 48.96 seconds +[May 1, 2026 - 02:22:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 209 : data loading time 0.00 +[May 1, 2026 - 02:22:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 209 : avg forward pass time 13.56 +[May 1, 2026 - 02:22:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 209 : backward pass time 10.14 +[May 1, 2026 - 02:22:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 209 : optimizer step time 1.57 +[May 1, 2026 - 02:22:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 209-------------------- +[May 1, 2026 - 02:22:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 02:22:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 02:22:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:22:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 02:25:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450175 41.608429 41.321701 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.086426 136.089844 136.076172 +util 66.000000 70.000000 62.000000 +[May 1, 2026 - 02:25:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 210 : avg iteration time 163.31 seconds +[May 1, 2026 - 02:25:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 210 : data loading time 0.00 +[May 1, 2026 - 02:25:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 210 : avg forward pass time 73.05 +[May 1, 2026 - 02:25:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 210 : backward pass time 8.43 +[May 1, 2026 - 02:25:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 210 : optimizer step time 0.39 +[May 1, 2026 - 02:25:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 210-------------------- +[May 1, 2026 - 02:25:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 3621475.4531 iter count: 1.0 +[May 1, 2026 - 02:25:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6165 iter count: 1.0 +[May 1, 2026 - 02:25:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 3621475.4531iter count: 1.0 +[May 1, 2026 - 02:25:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:25:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.7425iter count: 1.0 +[May 1, 2026 - 02:25:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.4069 iter count: 1.0 +[May 1, 2026 - 02:25:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 9.5746 iter count: 1.0 +[May 1, 2026 - 02:25:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 6592216.0000iter count: 1.0 +[May 1, 2026 - 02:25:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 2.2567 iter count: 1.0 +[May 1, 2026 - 02:25:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 207186096.0000iter count: 1.0 +[May 1, 2026 - 02:25:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6165iter count: 1.0 +[May 1, 2026 - 02:25:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.2287 iter count: 1.0 +[May 1, 2026 - 02:25:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 4709.6592iter count: 1.0 +[May 1, 2026 - 02:25:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.6062 iter count: 1.0 +[May 1, 2026 - 02:25:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 10020.8701iter count: 1.0 +[May 1, 2026 - 02:25:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.2387 iter count: 1.0 +[May 1, 2026 - 02:25:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 35503.7695iter count: 1.0 +[May 1, 2026 - 02:25:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 2.4062 iter count: 1.0 +[May 1, 2026 - 02:25:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 123.0205iter count: 1.0 +[May 1, 2026 - 02:25:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 1, 2026 - 02:25:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:25:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 02:26:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450181 41.608406 41.321701 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.540039 67.542969 67.531250 +util 92.250000 96.000000 88.000000 +[May 1, 2026 - 02:26:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 211 : avg iteration time 47.94 seconds +[May 1, 2026 - 02:26:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 211 : data loading time 0.00 +[May 1, 2026 - 02:26:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 211 : avg forward pass time 13.03 +[May 1, 2026 - 02:26:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 211 : backward pass time 10.14 +[May 1, 2026 - 02:26:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 211 : optimizer step time 1.57 +[May 1, 2026 - 02:26:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 211-------------------- +[May 1, 2026 - 02:26:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 02:26:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 02:26:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:26:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 02:27:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450375 41.609196 41.321716 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.442383 67.609375 67.386719 +util 94.500000 100.000000 88.000000 +[May 1, 2026 - 02:27:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 212 : avg iteration time 48.93 seconds +[May 1, 2026 - 02:27:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 212 : data loading time 0.00 +[May 1, 2026 - 02:27:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 212 : avg forward pass time 13.54 +[May 1, 2026 - 02:27:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 212 : backward pass time 10.13 +[May 1, 2026 - 02:27:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 212 : optimizer step time 1.57 +[May 1, 2026 - 02:27:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 212-------------------- +[May 1, 2026 - 02:27:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 1, 2026 - 02:27:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 1, 2026 - 02:27:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:27:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 02:28:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450437 41.609413 41.321701 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.422852 67.531250 67.386719 +util 95.250000 100.000000 89.000000 +[May 1, 2026 - 02:28:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 213 : avg iteration time 43.61 seconds +[May 1, 2026 - 02:28:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 213 : data loading time 0.00 +[May 1, 2026 - 02:28:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 213 : avg forward pass time 10.87 +[May 1, 2026 - 02:28:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 213 : backward pass time 10.16 +[May 1, 2026 - 02:28:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 213 : optimizer step time 1.57 +[May 1, 2026 - 02:28:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 213-------------------- +[May 1, 2026 - 02:28:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 1, 2026 - 02:28:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 1, 2026 - 02:28:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:28:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 02:28:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450491 41.609650 41.321701 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.432617 67.570312 67.386719 +util 96.500000 100.000000 89.000000 +[May 1, 2026 - 02:28:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 214 : avg iteration time 48.96 seconds +[May 1, 2026 - 02:28:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 214 : data loading time 0.00 +[May 1, 2026 - 02:28:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 214 : avg forward pass time 13.54 +[May 1, 2026 - 02:28:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 214 : backward pass time 10.15 +[May 1, 2026 - 02:28:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 214 : optimizer step time 1.58 +[May 1, 2026 - 02:28:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 214-------------------- +[May 1, 2026 - 02:28:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 02:28:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 02:28:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:28:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 02:31:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450097 41.608074 41.321701 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 135.998535 136.017578 135.992188 +util 66.750000 71.000000 63.000000 +[May 1, 2026 - 02:31:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 215 : avg iteration time 165.56 seconds +[May 1, 2026 - 02:31:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 215 : data loading time 0.00 +[May 1, 2026 - 02:31:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 215 : avg forward pass time 74.18 +[May 1, 2026 - 02:31:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 215 : backward pass time 8.40 +[May 1, 2026 - 02:31:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 215 : optimizer step time 0.39 +[May 1, 2026 - 02:31:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 215-------------------- +[May 1, 2026 - 02:31:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 276356.3174 iter count: 1.0 +[May 1, 2026 - 02:31:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6747 iter count: 1.0 +[May 1, 2026 - 02:31:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 276356.3174iter count: 1.0 +[May 1, 2026 - 02:31:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:31:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.8770iter count: 1.0 +[May 1, 2026 - 02:31:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.6848 iter count: 1.0 +[May 1, 2026 - 02:31:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.6511 iter count: 1.0 +[May 1, 2026 - 02:31:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 382399.8750iter count: 1.0 +[May 1, 2026 - 02:31:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 3.9335 iter count: 1.0 +[May 1, 2026 - 02:31:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 4422179.0000iter count: 1.0 +[May 1, 2026 - 02:31:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6747iter count: 1.0 +[May 1, 2026 - 02:31:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 6.1433 iter count: 1.0 +[May 1, 2026 - 02:31:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 216790.3281iter count: 1.0 +[May 1, 2026 - 02:31:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.6007 iter count: 1.0 +[May 1, 2026 - 02:31:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 9910.3506iter count: 1.0 +[May 1, 2026 - 02:31:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 1.7148 iter count: 1.0 +[May 1, 2026 - 02:31:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 30.8645 iter count: 1.0 +[May 1, 2026 - 02:31:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.1006 iter count: 1.0 +[May 1, 2026 - 02:31:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 199042.7344iter count: 1.0 +[May 1, 2026 - 02:31:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 1, 2026 - 02:31:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:31:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 02:32:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450158 41.608292 41.321701 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.359375 67.863281 67.191406 +util 95.750000 100.000000 88.000000 +[May 1, 2026 - 02:32:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 216 : avg iteration time 47.82 seconds +[May 1, 2026 - 02:32:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 216 : data loading time 0.00 +[May 1, 2026 - 02:32:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 216 : avg forward pass time 13.00 +[May 1, 2026 - 02:32:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 216 : backward pass time 10.13 +[May 1, 2026 - 02:32:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 216 : optimizer step time 1.56 +[May 1, 2026 - 02:32:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 216-------------------- +[May 1, 2026 - 02:32:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 1, 2026 - 02:32:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 1, 2026 - 02:32:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:32:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 02:33:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450464 41.609516 41.321701 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.325195 67.843750 67.152344 +util 93.250000 99.000000 87.000000 +[May 1, 2026 - 02:33:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 217 : avg iteration time 46.22 seconds +[May 1, 2026 - 02:33:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 217 : data loading time 0.00 +[May 1, 2026 - 02:33:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 217 : avg forward pass time 12.19 +[May 1, 2026 - 02:33:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 217 : backward pass time 10.14 +[May 1, 2026 - 02:33:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 217 : optimizer step time 1.56 +[May 1, 2026 - 02:33:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 217-------------------- +[May 1, 2026 - 02:33:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 02:33:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 02:33:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:33:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 02:34:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450602 41.610046 41.321701 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.325195 67.726562 67.191406 +util 93.250000 100.000000 88.000000 +[May 1, 2026 - 02:34:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 218 : avg iteration time 49.06 seconds +[May 1, 2026 - 02:34:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 218 : data loading time 0.00 +[May 1, 2026 - 02:34:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 218 : avg forward pass time 13.60 +[May 1, 2026 - 02:34:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 218 : backward pass time 10.13 +[May 1, 2026 - 02:34:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 218 : optimizer step time 1.58 +[May 1, 2026 - 02:34:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 218-------------------- +[May 1, 2026 - 02:34:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 02:34:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 02:34:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:34:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 02:34:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450494 41.609634 41.321701 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.745117 67.765625 67.738281 +util 95.000000 100.000000 89.000000 +[May 1, 2026 - 02:34:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 219 : avg iteration time 46.37 seconds +[May 1, 2026 - 02:34:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 219 : data loading time 0.00 +[May 1, 2026 - 02:34:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 219 : avg forward pass time 12.26 +[May 1, 2026 - 02:34:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 219 : backward pass time 10.16 +[May 1, 2026 - 02:34:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 219 : optimizer step time 1.57 +[May 1, 2026 - 02:34:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 219-------------------- +[May 1, 2026 - 02:34:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 02:34:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 02:34:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:34:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 02:37:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450195 41.608307 41.321808 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.104492 136.154297 136.087891 +util 61.750000 66.000000 58.000000 +[May 1, 2026 - 02:37:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 220 : avg iteration time 165.69 seconds +[May 1, 2026 - 02:37:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 220 : data loading time 0.00 +[May 1, 2026 - 02:37:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 220 : avg forward pass time 74.23 +[May 1, 2026 - 02:37:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 220 : backward pass time 8.43 +[May 1, 2026 - 02:37:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 220 : optimizer step time 0.42 +[May 1, 2026 - 02:37:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 220-------------------- +[May 1, 2026 - 02:37:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 225322.5088 iter count: 1.0 +[May 1, 2026 - 02:37:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6333 iter count: 1.0 +[May 1, 2026 - 02:37:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 225322.5088iter count: 1.0 +[May 1, 2026 - 02:37:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:37:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.7316iter count: 1.0 +[May 1, 2026 - 02:37:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.2847 iter count: 1.0 +[May 1, 2026 - 02:37:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.6037 iter count: 1.0 +[May 1, 2026 - 02:37:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 364789.6562iter count: 1.0 +[May 1, 2026 - 02:37:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 13.0577iter count: 1.0 +[May 1, 2026 - 02:37:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 4022379.2500iter count: 1.0 +[May 1, 2026 - 02:37:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6333iter count: 1.0 +[May 1, 2026 - 02:37:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.2345 iter count: 1.0 +[May 1, 2026 - 02:37:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 35207.1211iter count: 1.0 +[May 1, 2026 - 02:37:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 7.6037 iter count: 1.0 +[May 1, 2026 - 02:37:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 4022379.2500iter count: 1.0 +[May 1, 2026 - 02:37:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 2.7039 iter count: 1.0 +[May 1, 2026 - 02:37:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 223.1424iter count: 1.0 +[May 1, 2026 - 02:37:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.1898 iter count: 1.0 +[May 1, 2026 - 02:37:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 237902.6406iter count: 1.0 +[May 1, 2026 - 02:37:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 02:37:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:37:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 02:38:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450199 41.608330 41.321808 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.237305 67.433594 67.171875 +util 96.500000 99.000000 93.000000 +[May 1, 2026 - 02:38:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 221 : avg iteration time 45.21 seconds +[May 1, 2026 - 02:38:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 221 : data loading time 0.00 +[May 1, 2026 - 02:38:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 221 : avg forward pass time 11.68 +[May 1, 2026 - 02:38:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 221 : backward pass time 10.15 +[May 1, 2026 - 02:38:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 221 : optimizer step time 1.57 +[May 1, 2026 - 02:38:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 221-------------------- +[May 1, 2026 - 02:38:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 02:38:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 02:38:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:38:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 02:39:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450496 41.609531 41.321808 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.232422 67.414062 67.171875 +util 94.000000 100.000000 89.000000 +[May 1, 2026 - 02:39:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 222 : avg iteration time 48.93 seconds +[May 1, 2026 - 02:39:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 222 : data loading time 0.00 +[May 1, 2026 - 02:39:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 222 : avg forward pass time 13.54 +[May 1, 2026 - 02:39:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 222 : backward pass time 10.14 +[May 1, 2026 - 02:39:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 222 : optimizer step time 1.55 +[May 1, 2026 - 02:39:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 222-------------------- +[May 1, 2026 - 02:39:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 1, 2026 - 02:39:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 1, 2026 - 02:39:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:39:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 02:39:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450500 41.609531 41.321808 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.247070 67.414062 67.191406 +util 93.750000 99.000000 87.000000 +[May 1, 2026 - 02:39:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 223 : avg iteration time 48.93 seconds +[May 1, 2026 - 02:39:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 223 : data loading time 0.00 +[May 1, 2026 - 02:39:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 223 : avg forward pass time 13.53 +[May 1, 2026 - 02:39:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 223 : backward pass time 10.15 +[May 1, 2026 - 02:39:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 223 : optimizer step time 1.57 +[May 1, 2026 - 02:39:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 223-------------------- +[May 1, 2026 - 02:39:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 1, 2026 - 02:39:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 1, 2026 - 02:39:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:39:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 02:40:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450428 41.609241 41.321808 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.627930 67.660156 67.531250 +util 92.250000 99.000000 87.000000 +[May 1, 2026 - 02:40:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 224 : avg iteration time 43.64 seconds +[May 1, 2026 - 02:40:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 224 : data loading time 0.00 +[May 1, 2026 - 02:40:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 224 : avg forward pass time 10.90 +[May 1, 2026 - 02:40:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 224 : backward pass time 10.12 +[May 1, 2026 - 02:40:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 224 : optimizer step time 1.58 +[May 1, 2026 - 02:40:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 224-------------------- +[May 1, 2026 - 02:40:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 02:40:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 02:40:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:40:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 02:43:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450235 41.608448 41.321808 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.031250 136.037109 136.029297 +util 64.500000 69.000000 61.000000 +[May 1, 2026 - 02:43:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 225 : avg iteration time 171.48 seconds +[May 1, 2026 - 02:43:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 225 : data loading time 0.00 +[May 1, 2026 - 02:43:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 225 : avg forward pass time 77.14 +[May 1, 2026 - 02:43:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 225 : backward pass time 8.45 +[May 1, 2026 - 02:43:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 225 : optimizer step time 0.39 +[May 1, 2026 - 02:43:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 225-------------------- +[May 1, 2026 - 02:43:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 233042.3633 iter count: 1.0 +[May 1, 2026 - 02:43:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6250 iter count: 1.0 +[May 1, 2026 - 02:43:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 233042.3633iter count: 1.0 +[May 1, 2026 - 02:43:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:43:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.8586iter count: 1.0 +[May 1, 2026 - 02:43:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.5944 iter count: 1.0 +[May 1, 2026 - 02:43:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.9103 iter count: 1.0 +[May 1, 2026 - 02:43:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 398400.8438iter count: 1.0 +[May 1, 2026 - 02:43:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 24.2587iter count: 1.0 +[May 1, 2026 - 02:43:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 7427349.5000iter count: 1.0 +[May 1, 2026 - 02:43:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6250iter count: 1.0 +[May 1, 2026 - 02:43:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 1.5944 iter count: 1.0 +[May 1, 2026 - 02:43:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 24.2587 iter count: 1.0 +[May 1, 2026 - 02:43:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.5847 iter count: 1.0 +[May 1, 2026 - 02:43:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 70926.2188iter count: 1.0 +[May 1, 2026 - 02:43:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.6734 iter count: 1.0 +[May 1, 2026 - 02:43:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 11461.9746iter count: 1.0 +[May 1, 2026 - 02:43:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.9005 iter count: 1.0 +[May 1, 2026 - 02:43:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 18053.4336iter count: 1.0 +[May 1, 2026 - 02:43:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 02:43:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:43:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 02:44:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450234 41.608444 41.321808 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.750000 67.855469 67.433594 +util 97.000000 99.000000 94.000000 +[May 1, 2026 - 02:44:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 226 : avg iteration time 50.43 seconds +[May 1, 2026 - 02:44:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 226 : data loading time 0.00 +[May 1, 2026 - 02:44:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 226 : avg forward pass time 14.29 +[May 1, 2026 - 02:44:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 226 : backward pass time 10.13 +[May 1, 2026 - 02:44:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 226 : optimizer step time 1.58 +[May 1, 2026 - 02:44:21 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 226-------------------- +[May 1, 2026 - 02:44:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 1, 2026 - 02:44:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 1, 2026 - 02:44:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:44:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 02:45:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450243 41.608463 41.321808 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.706055 67.796875 67.433594 +util 93.250000 100.000000 89.000000 +[May 1, 2026 - 02:45:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 227 : avg iteration time 48.89 seconds +[May 1, 2026 - 02:45:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 227 : data loading time 0.00 +[May 1, 2026 - 02:45:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 227 : avg forward pass time 13.50 +[May 1, 2026 - 02:45:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 227 : backward pass time 10.17 +[May 1, 2026 - 02:45:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 227 : optimizer step time 1.57 +[May 1, 2026 - 02:45:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 227-------------------- +[May 1, 2026 - 02:45:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 1, 2026 - 02:45:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 1, 2026 - 02:45:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:45:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 02:45:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450240 41.608452 41.321819 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.750000 67.855469 67.433594 +util 98.000000 100.000000 95.000000 +[May 1, 2026 - 02:45:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 228 : avg iteration time 43.52 seconds +[May 1, 2026 - 02:45:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 228 : data loading time 0.00 +[May 1, 2026 - 02:45:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 228 : avg forward pass time 10.83 +[May 1, 2026 - 02:45:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 228 : backward pass time 10.16 +[May 1, 2026 - 02:45:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 228 : optimizer step time 1.55 +[May 1, 2026 - 02:45:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 228-------------------- +[May 1, 2026 - 02:45:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 02:45:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 02:45:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:45:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 02:46:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450369 41.608944 41.321819 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.750000 67.855469 67.433594 +util 94.250000 98.000000 90.000000 +[May 1, 2026 - 02:46:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 229 : avg iteration time 48.94 seconds +[May 1, 2026 - 02:46:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 229 : data loading time 0.00 +[May 1, 2026 - 02:46:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 229 : avg forward pass time 13.54 +[May 1, 2026 - 02:46:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 229 : backward pass time 10.14 +[May 1, 2026 - 02:46:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 229 : optimizer step time 1.57 +[May 1, 2026 - 02:46:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 229-------------------- +[May 1, 2026 - 02:46:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 1, 2026 - 02:46:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 1, 2026 - 02:46:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:46:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 02:49:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450233 41.608284 41.321838 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.062012 136.095703 136.050781 +util 61.250000 65.000000 57.000000 +[May 1, 2026 - 02:49:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 230 : avg iteration time 163.19 seconds +[May 1, 2026 - 02:49:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 230 : data loading time 0.00 +[May 1, 2026 - 02:49:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 230 : avg forward pass time 72.98 +[May 1, 2026 - 02:49:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 230 : backward pass time 8.44 +[May 1, 2026 - 02:49:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 230 : optimizer step time 0.40 +[May 1, 2026 - 02:49:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 230-------------------- +[May 1, 2026 - 02:49:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 112839.4746 iter count: 1.0 +[May 1, 2026 - 02:49:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6373 iter count: 1.0 +[May 1, 2026 - 02:49:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 112839.4746iter count: 1.0 +[May 1, 2026 - 02:49:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:49:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.5924iter count: 1.0 +[May 1, 2026 - 02:49:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.5905 iter count: 1.0 +[May 1, 2026 - 02:49:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.4960 iter count: 1.0 +[May 1, 2026 - 02:49:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 179516.4844iter count: 1.0 +[May 1, 2026 - 02:49:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 24.0697iter count: 1.0 +[May 1, 2026 - 02:49:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 3242872.5000iter count: 1.0 +[May 1, 2026 - 02:49:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6373iter count: 1.0 +[May 1, 2026 - 02:49:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.3190 iter count: 1.0 +[May 1, 2026 - 02:49:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 5641.9189iter count: 1.0 +[May 1, 2026 - 02:49:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.5091 iter count: 1.0 +[May 1, 2026 - 02:49:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 60974.7578iter count: 1.0 +[May 1, 2026 - 02:49:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.4275 iter count: 1.0 +[May 1, 2026 - 02:49:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 948.6968iter count: 1.0 +[May 1, 2026 - 02:49:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.7513 iter count: 1.0 +[May 1, 2026 - 02:49:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 13395.4482iter count: 1.0 +[May 1, 2026 - 02:49:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 1, 2026 - 02:49:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:49:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 02:50:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450231 41.608273 41.321819 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 69.302734 69.886719 67.550781 +util 94.000000 98.000000 90.000000 +[May 1, 2026 - 02:50:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 231 : avg iteration time 50.61 seconds +[May 1, 2026 - 02:50:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 231 : data loading time 0.00 +[May 1, 2026 - 02:50:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 231 : avg forward pass time 14.37 +[May 1, 2026 - 02:50:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 231 : backward pass time 10.15 +[May 1, 2026 - 02:50:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 231 : optimizer step time 1.56 +[May 1, 2026 - 02:50:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 231-------------------- +[May 1, 2026 - 02:50:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 02:50:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 02:50:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:50:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 02:51:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450303 41.608524 41.321819 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.276367 67.531250 67.191406 +util 93.750000 100.000000 89.000000 +[May 1, 2026 - 02:51:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 232 : avg iteration time 46.16 seconds +[May 1, 2026 - 02:51:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 232 : data loading time 0.00 +[May 1, 2026 - 02:51:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 232 : avg forward pass time 12.17 +[May 1, 2026 - 02:51:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 232 : backward pass time 10.13 +[May 1, 2026 - 02:51:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 232 : optimizer step time 1.59 +[May 1, 2026 - 02:51:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 232-------------------- +[May 1, 2026 - 02:51:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0086 iter count: 1.0 +[May 1, 2026 - 02:51:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 1, 2026 - 02:51:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:51:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 02:51:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450304 41.608524 41.321823 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.481445 67.503906 67.414062 +util 93.750000 100.000000 87.000000 +[May 1, 2026 - 02:51:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 233 : avg iteration time 43.53 seconds +[May 1, 2026 - 02:51:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 233 : data loading time 0.00 +[May 1, 2026 - 02:51:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 233 : avg forward pass time 10.87 +[May 1, 2026 - 02:51:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 233 : backward pass time 10.11 +[May 1, 2026 - 02:51:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 233 : optimizer step time 1.56 +[May 1, 2026 - 02:51:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 233-------------------- +[May 1, 2026 - 02:51:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0082 iter count: 1.0 +[May 1, 2026 - 02:51:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[May 1, 2026 - 02:51:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:51:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 02:52:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450329 41.608524 41.321918 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.491211 67.503906 67.453125 +util 94.000000 100.000000 89.000000 +[May 1, 2026 - 02:52:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 234 : avg iteration time 46.27 seconds +[May 1, 2026 - 02:52:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 234 : data loading time 0.00 +[May 1, 2026 - 02:52:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 234 : avg forward pass time 12.19 +[May 1, 2026 - 02:52:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 234 : backward pass time 10.15 +[May 1, 2026 - 02:52:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 234 : optimizer step time 1.59 +[May 1, 2026 - 02:52:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 234-------------------- +[May 1, 2026 - 02:52:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[May 1, 2026 - 02:52:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 1, 2026 - 02:52:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:52:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 02:55:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450000 41.607170 41.321911 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.027832 136.031250 136.017578 +util 65.250000 70.000000 62.000000 +[May 1, 2026 - 02:55:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 235 : avg iteration time 163.07 seconds +[May 1, 2026 - 02:55:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 235 : data loading time 0.00 +[May 1, 2026 - 02:55:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 235 : avg forward pass time 72.92 +[May 1, 2026 - 02:55:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 235 : backward pass time 8.42 +[May 1, 2026 - 02:55:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 235 : optimizer step time 0.40 +[May 1, 2026 - 02:55:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 235-------------------- +[May 1, 2026 - 02:55:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 513101.7737 iter count: 1.0 +[May 1, 2026 - 02:55:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6160 iter count: 1.0 +[May 1, 2026 - 02:55:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 513101.7737iter count: 1.0 +[May 1, 2026 - 02:55:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:55:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.5911iter count: 1.0 +[May 1, 2026 - 02:55:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.4717 iter count: 1.0 +[May 1, 2026 - 02:55:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.3032 iter count: 1.0 +[May 1, 2026 - 02:55:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 868781.5625iter count: 1.0 +[May 1, 2026 - 02:55:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 2.5686 iter count: 1.0 +[May 1, 2026 - 02:55:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 16295935.0000iter count: 1.0 +[May 1, 2026 - 02:55:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6160iter count: 1.0 +[May 1, 2026 - 02:55:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 6.6026 iter count: 1.0 +[May 1, 2026 - 02:55:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 543134.5625iter count: 1.0 +[May 1, 2026 - 02:55:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.8209 iter count: 1.0 +[May 1, 2026 - 02:55:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 15394.7344iter count: 1.0 +[May 1, 2026 - 02:55:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 2.8740 iter count: 1.0 +[May 1, 2026 - 02:55:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 313.5800iter count: 1.0 +[May 1, 2026 - 02:55:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 2.6749 iter count: 1.0 +[May 1, 2026 - 02:55:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 210.5730iter count: 1.0 +[May 1, 2026 - 02:55:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0080 iter count: 1.0 +[May 1, 2026 - 02:55:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:55:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 02:56:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449988 41.607170 41.321911 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.495605 67.523438 67.412109 +util 95.750000 100.000000 90.000000 +[May 1, 2026 - 02:56:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 236 : avg iteration time 45.31 seconds +[May 1, 2026 - 02:56:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 236 : data loading time 0.00 +[May 1, 2026 - 02:56:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 236 : avg forward pass time 11.73 +[May 1, 2026 - 02:56:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 236 : backward pass time 10.14 +[May 1, 2026 - 02:56:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 236 : optimizer step time 1.57 +[May 1, 2026 - 02:56:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 236-------------------- +[May 1, 2026 - 02:56:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0082 iter count: 1.0 +[May 1, 2026 - 02:56:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[May 1, 2026 - 02:56:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:56:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 02:56:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450261 41.608147 41.322025 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.251465 67.431641 67.191406 +util 94.000000 99.000000 88.000000 +[May 1, 2026 - 02:56:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 237 : avg iteration time 43.57 seconds +[May 1, 2026 - 02:56:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 237 : data loading time 0.00 +[May 1, 2026 - 02:56:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 237 : avg forward pass time 10.86 +[May 1, 2026 - 02:56:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 237 : backward pass time 10.14 +[May 1, 2026 - 02:56:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 237 : optimizer step time 1.55 +[May 1, 2026 - 02:56:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 237-------------------- +[May 1, 2026 - 02:56:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0084 iter count: 1.0 +[May 1, 2026 - 02:56:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 1, 2026 - 02:56:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:56:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 02:57:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450411 41.608643 41.322121 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.300293 67.568359 67.210938 +util 96.000000 99.000000 88.000000 +[May 1, 2026 - 02:57:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 238 : avg iteration time 48.98 seconds +[May 1, 2026 - 02:57:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 238 : data loading time 0.00 +[May 1, 2026 - 02:57:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 238 : avg forward pass time 13.57 +[May 1, 2026 - 02:57:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 238 : backward pass time 10.14 +[May 1, 2026 - 02:57:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 238 : optimizer step time 1.57 +[May 1, 2026 - 02:57:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 238-------------------- +[May 1, 2026 - 02:57:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 02:57:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 02:57:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:57:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 02:58:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450327 41.608349 41.322079 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.266113 67.548828 67.171875 +util 98.000000 100.000000 95.000000 +[May 1, 2026 - 02:58:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 239 : avg iteration time 48.98 seconds +[May 1, 2026 - 02:58:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 239 : data loading time 0.00 +[May 1, 2026 - 02:58:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 239 : avg forward pass time 13.55 +[May 1, 2026 - 02:58:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 239 : backward pass time 10.16 +[May 1, 2026 - 02:58:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 239 : optimizer step time 1.56 +[May 1, 2026 - 02:58:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 239-------------------- +[May 1, 2026 - 02:58:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 02:58:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 02:58:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 02:58:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 03:01:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450138 41.607540 41.322075 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.080078 136.115234 136.068359 +util 65.500000 71.000000 59.000000 +[May 1, 2026 - 03:01:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 240 : avg iteration time 165.89 seconds +[May 1, 2026 - 03:01:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 240 : data loading time 0.00 +[May 1, 2026 - 03:01:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 240 : avg forward pass time 74.33 +[May 1, 2026 - 03:01:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 240 : backward pass time 8.46 +[May 1, 2026 - 03:01:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 240 : optimizer step time 0.39 +[May 1, 2026 - 03:01:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 240-------------------- +[May 1, 2026 - 03:01:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 645557.1504 iter count: 1.0 +[May 1, 2026 - 03:01:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6688 iter count: 1.0 +[May 1, 2026 - 03:01:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 645557.1504iter count: 1.0 +[May 1, 2026 - 03:01:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:01:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.5693iter count: 1.0 +[May 1, 2026 - 03:01:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.0513 iter count: 1.0 +[May 1, 2026 - 03:01:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.3928 iter count: 1.0 +[May 1, 2026 - 03:01:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 891722.6250iter count: 1.0 +[May 1, 2026 - 03:01:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 8.1878 iter count: 1.0 +[May 1, 2026 - 03:01:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 19493082.0000iter count: 1.0 +[May 1, 2026 - 03:01:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6688iter count: 1.0 +[May 1, 2026 - 03:01:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.9197 iter count: 1.0 +[May 1, 2026 - 03:01:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 18758.0703iter count: 1.0 +[May 1, 2026 - 03:01:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.5267 iter count: 1.0 +[May 1, 2026 - 03:01:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 8547.2080iter count: 1.0 +[May 1, 2026 - 03:01:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.0137 iter count: 1.0 +[May 1, 2026 - 03:01:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 3064.0374iter count: 1.0 +[May 1, 2026 - 03:01:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.4820 iter count: 1.0 +[May 1, 2026 - 03:01:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 7816.1641iter count: 1.0 +[May 1, 2026 - 03:01:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 03:01:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:01:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 03:01:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450143 41.607555 41.322121 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.651855 67.699219 67.509766 +util 93.000000 97.000000 89.000000 +[May 1, 2026 - 03:01:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 241 : avg iteration time 47.86 seconds +[May 1, 2026 - 03:01:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 241 : data loading time 0.00 +[May 1, 2026 - 03:01:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 241 : avg forward pass time 13.01 +[May 1, 2026 - 03:01:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 241 : backward pass time 10.14 +[May 1, 2026 - 03:01:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 241 : optimizer step time 1.56 +[May 1, 2026 - 03:01:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 241-------------------- +[May 1, 2026 - 03:01:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 1, 2026 - 03:01:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 1, 2026 - 03:01:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:01:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 03:02:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450377 41.608551 41.322075 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.593262 67.660156 67.392578 +util 93.750000 98.000000 90.000000 +[May 1, 2026 - 03:02:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 242 : avg iteration time 43.61 seconds +[May 1, 2026 - 03:02:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 242 : data loading time 0.00 +[May 1, 2026 - 03:02:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 242 : avg forward pass time 10.86 +[May 1, 2026 - 03:02:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 242 : backward pass time 10.14 +[May 1, 2026 - 03:02:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 242 : optimizer step time 1.57 +[May 1, 2026 - 03:02:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 242-------------------- +[May 1, 2026 - 03:02:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 03:02:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 03:02:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:02:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 03:03:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450442 41.608795 41.322094 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.798340 67.933594 67.392578 +util 96.500000 99.000000 93.000000 +[May 1, 2026 - 03:03:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 243 : avg iteration time 46.44 seconds +[May 1, 2026 - 03:03:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 243 : data loading time 0.00 +[May 1, 2026 - 03:03:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 243 : avg forward pass time 12.29 +[May 1, 2026 - 03:03:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 243 : backward pass time 10.14 +[May 1, 2026 - 03:03:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 243 : optimizer step time 1.58 +[May 1, 2026 - 03:03:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 243-------------------- +[May 1, 2026 - 03:03:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0079 iter count: 1.0 +[May 1, 2026 - 03:03:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0079 iter count: 1.0 +[May 1, 2026 - 03:03:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:03:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 03:04:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450446 41.608799 41.322098 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.637207 67.718750 67.392578 +util 94.500000 99.000000 91.000000 +[May 1, 2026 - 03:04:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 244 : avg iteration time 43.57 seconds +[May 1, 2026 - 03:04:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 244 : data loading time 0.00 +[May 1, 2026 - 03:04:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 244 : avg forward pass time 10.84 +[May 1, 2026 - 03:04:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 244 : backward pass time 10.16 +[May 1, 2026 - 03:04:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 244 : optimizer step time 1.57 +[May 1, 2026 - 03:04:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 244-------------------- +[May 1, 2026 - 03:04:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 03:04:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 03:04:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:04:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 03:06:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450027 41.607105 41.322102 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.031250 136.037109 136.029297 +util 62.500000 69.000000 55.000000 +[May 1, 2026 - 03:06:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 245 : avg iteration time 163.30 seconds +[May 1, 2026 - 03:06:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 245 : data loading time 0.00 +[May 1, 2026 - 03:06:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 245 : avg forward pass time 73.03 +[May 1, 2026 - 03:06:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 245 : backward pass time 8.45 +[May 1, 2026 - 03:06:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 245 : optimizer step time 0.39 +[May 1, 2026 - 03:06:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 245-------------------- +[May 1, 2026 - 03:06:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 47710.6562 iter count: 1.0 +[May 1, 2026 - 03:06:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6422 iter count: 1.0 +[May 1, 2026 - 03:06:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 47710.6562iter count: 1.0 +[May 1, 2026 - 03:06:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:06:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.2649iter count: 1.0 +[May 1, 2026 - 03:06:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.6245 iter count: 1.0 +[May 1, 2026 - 03:06:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.0909 iter count: 1.0 +[May 1, 2026 - 03:06:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 74155.1172iter count: 1.0 +[May 1, 2026 - 03:06:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 25.7625iter count: 1.0 +[May 1, 2026 - 03:06:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 1442475.7500iter count: 1.0 +[May 1, 2026 - 03:06:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6422iter count: 1.0 +[May 1, 2026 - 03:06:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 2.6341 iter count: 1.0 +[May 1, 2026 - 03:06:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 194.0541iter count: 1.0 +[May 1, 2026 - 03:06:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.9354 iter count: 1.0 +[May 1, 2026 - 03:06:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 2619.8403iter count: 1.0 +[May 1, 2026 - 03:06:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 7.0909 iter count: 1.0 +[May 1, 2026 - 03:06:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 1442475.7500iter count: 1.0 +[May 1, 2026 - 03:06:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.2994 iter count: 1.0 +[May 1, 2026 - 03:06:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 734.1987iter count: 1.0 +[May 1, 2026 - 03:06:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[May 1, 2026 - 03:06:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:06:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 03:07:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450084 41.607376 41.322079 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.520508 67.542969 67.453125 +util 95.750000 99.000000 92.000000 +[May 1, 2026 - 03:07:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 246 : avg iteration time 47.93 seconds +[May 1, 2026 - 03:07:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 246 : data loading time 0.00 +[May 1, 2026 - 03:07:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 246 : avg forward pass time 13.02 +[May 1, 2026 - 03:07:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 246 : backward pass time 10.13 +[May 1, 2026 - 03:07:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 246 : optimizer step time 1.58 +[May 1, 2026 - 03:07:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 246-------------------- +[May 1, 2026 - 03:07:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0101 iter count: 1.0 +[May 1, 2026 - 03:07:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 1, 2026 - 03:07:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:07:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 03:08:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450394 41.608601 41.322094 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.979980 69.476562 67.490234 +util 97.750000 100.000000 95.000000 +[May 1, 2026 - 03:08:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 247 : avg iteration time 46.24 seconds +[May 1, 2026 - 03:08:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 247 : data loading time 0.00 +[May 1, 2026 - 03:08:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 247 : avg forward pass time 12.22 +[May 1, 2026 - 03:08:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 247 : backward pass time 10.12 +[May 1, 2026 - 03:08:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 247 : optimizer step time 1.55 +[May 1, 2026 - 03:08:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 247-------------------- +[May 1, 2026 - 03:08:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 03:08:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 03:08:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:08:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 03:09:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450425 41.608589 41.322231 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.921387 69.417969 67.431641 +util 94.000000 99.000000 89.000000 +[May 1, 2026 - 03:09:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 248 : avg iteration time 48.90 seconds +[May 1, 2026 - 03:09:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 248 : data loading time 0.00 +[May 1, 2026 - 03:09:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 248 : avg forward pass time 13.54 +[May 1, 2026 - 03:09:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 248 : backward pass time 10.11 +[May 1, 2026 - 03:09:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 248 : optimizer step time 1.57 +[May 1, 2026 - 03:09:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 248-------------------- +[May 1, 2026 - 03:09:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 1, 2026 - 03:09:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 1, 2026 - 03:09:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:09:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 03:10:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450425 41.608589 41.322231 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.236816 67.490234 67.152344 +util 94.750000 99.000000 89.000000 +[May 1, 2026 - 03:10:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 249 : avg iteration time 46.36 seconds +[May 1, 2026 - 03:10:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 249 : data loading time 0.00 +[May 1, 2026 - 03:10:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 249 : avg forward pass time 12.25 +[May 1, 2026 - 03:10:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 249 : backward pass time 10.14 +[May 1, 2026 - 03:10:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 249 : optimizer step time 1.59 +[May 1, 2026 - 03:10:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 249-------------------- +[May 1, 2026 - 03:10:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0084 iter count: 1.0 +[May 1, 2026 - 03:10:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 1, 2026 - 03:10:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:10:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 03:12:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449989 41.606850 41.322220 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.036133 136.048828 135.998047 +util 66.000000 72.000000 59.000000 +[May 1, 2026 - 03:12:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 250 : avg iteration time 171.02 seconds +[May 1, 2026 - 03:12:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 250 : data loading time 0.00 +[May 1, 2026 - 03:12:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 250 : avg forward pass time 76.92 +[May 1, 2026 - 03:12:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 250 : backward pass time 8.40 +[May 1, 2026 - 03:12:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 250 : optimizer step time 0.39 +[May 1, 2026 - 03:12:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 250-------------------- +[May 1, 2026 - 03:12:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 297880.3789 iter count: 1.0 +[May 1, 2026 - 03:12:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6142 iter count: 1.0 +[May 1, 2026 - 03:12:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 297880.3789iter count: 1.0 +[May 1, 2026 - 03:12:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:12:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.8215iter count: 1.0 +[May 1, 2026 - 03:12:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.9888 iter count: 1.0 +[May 1, 2026 - 03:12:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.8952 iter count: 1.0 +[May 1, 2026 - 03:12:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 479565.2500iter count: 1.0 +[May 1, 2026 - 03:12:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 7.2249 iter count: 1.0 +[May 1, 2026 - 03:12:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 7205544.5000iter count: 1.0 +[May 1, 2026 - 03:12:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6142iter count: 1.0 +[May 1, 2026 - 03:12:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.1482 iter count: 1.0 +[May 1, 2026 - 03:12:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 4009.1299iter count: 1.0 +[May 1, 2026 - 03:12:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.6838 iter count: 1.0 +[May 1, 2026 - 03:12:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 1583.6884iter count: 1.0 +[May 1, 2026 - 03:12:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.1951 iter count: 1.0 +[May 1, 2026 - 03:12:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 4403.2935iter count: 1.0 +[May 1, 2026 - 03:12:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.2818 iter count: 1.0 +[May 1, 2026 - 03:12:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 38698.3711iter count: 1.0 +[May 1, 2026 - 03:12:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 03:12:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:12:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 03:13:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450123 41.607342 41.322231 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.647461 67.660156 67.609375 +util 95.750000 100.000000 91.000000 +[May 1, 2026 - 03:13:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 251 : avg iteration time 50.40 seconds +[May 1, 2026 - 03:13:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 251 : data loading time 0.00 +[May 1, 2026 - 03:13:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 251 : avg forward pass time 14.27 +[May 1, 2026 - 03:13:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 251 : backward pass time 10.15 +[May 1, 2026 - 03:13:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 251 : optimizer step time 1.57 +[May 1, 2026 - 03:13:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 251-------------------- +[May 1, 2026 - 03:13:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 03:13:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 03:13:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:13:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 03:14:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450436 41.608597 41.322247 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.505371 67.568359 67.484375 +util 94.000000 98.000000 90.000000 +[May 1, 2026 - 03:14:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 252 : avg iteration time 48.87 seconds +[May 1, 2026 - 03:14:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 252 : data loading time 0.00 +[May 1, 2026 - 03:14:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 252 : avg forward pass time 13.51 +[May 1, 2026 - 03:14:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 252 : backward pass time 10.15 +[May 1, 2026 - 03:14:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 252 : optimizer step time 1.56 +[May 1, 2026 - 03:14:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 252-------------------- +[May 1, 2026 - 03:14:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 1, 2026 - 03:14:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 1, 2026 - 03:14:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:14:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 03:15:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450492 41.608849 41.322220 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.495605 67.587891 67.464844 +util 97.500000 99.000000 95.000000 +[May 1, 2026 - 03:15:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 253 : avg iteration time 46.25 seconds +[May 1, 2026 - 03:15:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 253 : data loading time 0.00 +[May 1, 2026 - 03:15:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 253 : avg forward pass time 12.20 +[May 1, 2026 - 03:15:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 253 : backward pass time 10.15 +[May 1, 2026 - 03:15:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 253 : optimizer step time 1.57 +[May 1, 2026 - 03:15:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 253-------------------- +[May 1, 2026 - 03:15:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0086 iter count: 1.0 +[May 1, 2026 - 03:15:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 1, 2026 - 03:15:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:15:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 03:16:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450618 41.609352 41.322220 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.686035 67.718750 67.587891 +util 95.500000 100.000000 90.000000 +[May 1, 2026 - 03:16:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 254 : avg iteration time 46.27 seconds +[May 1, 2026 - 03:16:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 254 : data loading time 0.00 +[May 1, 2026 - 03:16:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 254 : avg forward pass time 12.20 +[May 1, 2026 - 03:16:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 254 : backward pass time 10.15 +[May 1, 2026 - 03:16:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 254 : optimizer step time 1.57 +[May 1, 2026 - 03:16:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 254-------------------- +[May 1, 2026 - 03:16:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0080 iter count: 1.0 +[May 1, 2026 - 03:16:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0080 iter count: 1.0 +[May 1, 2026 - 03:16:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:16:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 03:18:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.449977 41.606781 41.322220 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.071777 136.089844 136.017578 +util 62.750000 65.000000 59.000000 +[May 1, 2026 - 03:18:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 255 : avg iteration time 165.45 seconds +[May 1, 2026 - 03:18:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 255 : data loading time 0.00 +[May 1, 2026 - 03:18:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 255 : avg forward pass time 74.14 +[May 1, 2026 - 03:18:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 255 : backward pass time 8.41 +[May 1, 2026 - 03:18:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 255 : optimizer step time 0.40 +[May 1, 2026 - 03:18:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 255-------------------- +[May 1, 2026 - 03:18:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 224586.8770 iter count: 1.0 +[May 1, 2026 - 03:18:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6680 iter count: 1.0 +[May 1, 2026 - 03:18:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 224586.8770iter count: 1.0 +[May 1, 2026 - 03:18:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:18:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.4994iter count: 1.0 +[May 1, 2026 - 03:18:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.3212 iter count: 1.0 +[May 1, 2026 - 03:18:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.6654 iter count: 1.0 +[May 1, 2026 - 03:18:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 346978.5625iter count: 1.0 +[May 1, 2026 - 03:18:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 14.0469iter count: 1.0 +[May 1, 2026 - 03:18:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 4550751.5000iter count: 1.0 +[May 1, 2026 - 03:18:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6680iter count: 1.0 +[May 1, 2026 - 03:18:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 7.1813 iter count: 1.0 +[May 1, 2026 - 03:18:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 1728303.6250iter count: 1.0 +[May 1, 2026 - 03:18:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.9147 iter count: 1.0 +[May 1, 2026 - 03:18:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 2513.5579iter count: 1.0 +[May 1, 2026 - 03:18:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.7469 iter count: 1.0 +[May 1, 2026 - 03:18:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 1796.9994iter count: 1.0 +[May 1, 2026 - 03:18:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.0782 iter count: 1.0 +[May 1, 2026 - 03:18:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 471.7028iter count: 1.0 +[May 1, 2026 - 03:18:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 1, 2026 - 03:18:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:18:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 03:19:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450055 41.607056 41.322220 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.202637 67.470703 67.113281 +util 95.750000 100.000000 90.000000 +[May 1, 2026 - 03:19:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 256 : avg iteration time 47.75 seconds +[May 1, 2026 - 03:19:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 256 : data loading time 0.00 +[May 1, 2026 - 03:19:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 256 : avg forward pass time 12.97 +[May 1, 2026 - 03:19:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 256 : backward pass time 10.15 +[May 1, 2026 - 03:19:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 256 : optimizer step time 1.55 +[May 1, 2026 - 03:19:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 256-------------------- +[May 1, 2026 - 03:19:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 03:19:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 03:19:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:19:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 03:20:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450420 41.608540 41.322220 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.241699 67.568359 67.132812 +util 97.750000 99.000000 96.000000 +[May 1, 2026 - 03:20:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 257 : avg iteration time 43.52 seconds +[May 1, 2026 - 03:20:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 257 : data loading time 0.00 +[May 1, 2026 - 03:20:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 257 : avg forward pass time 10.83 +[May 1, 2026 - 03:20:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 257 : backward pass time 10.13 +[May 1, 2026 - 03:20:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 257 : optimizer step time 1.59 +[May 1, 2026 - 03:20:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 257-------------------- +[May 1, 2026 - 03:20:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0083 iter count: 1.0 +[May 1, 2026 - 03:20:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[May 1, 2026 - 03:20:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:20:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 03:21:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450484 41.608768 41.322247 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.236816 67.490234 67.152344 +util 93.000000 97.000000 89.000000 +[May 1, 2026 - 03:21:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 258 : avg iteration time 48.93 seconds +[May 1, 2026 - 03:21:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 258 : data loading time 0.00 +[May 1, 2026 - 03:21:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 258 : avg forward pass time 13.53 +[May 1, 2026 - 03:21:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 258 : backward pass time 10.15 +[May 1, 2026 - 03:21:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 258 : optimizer step time 1.57 +[May 1, 2026 - 03:21:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 258-------------------- +[May 1, 2026 - 03:21:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 1, 2026 - 03:21:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 1, 2026 - 03:21:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:21:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 03:21:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450624 41.609261 41.322273 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.661621 67.718750 67.490234 +util 96.500000 99.000000 93.000000 +[May 1, 2026 - 03:21:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 259 : avg iteration time 46.25 seconds +[May 1, 2026 - 03:21:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 259 : data loading time 0.00 +[May 1, 2026 - 03:21:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 259 : avg forward pass time 12.20 +[May 1, 2026 - 03:21:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 259 : backward pass time 10.13 +[May 1, 2026 - 03:21:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 259 : optimizer step time 1.57 +[May 1, 2026 - 03:21:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 259-------------------- +[May 1, 2026 - 03:21:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 03:21:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 03:21:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:21:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 03:24:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450051 41.606953 41.322273 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.104492 136.107422 136.095703 +util 66.000000 72.000000 59.000000 +[May 1, 2026 - 03:24:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 260 : avg iteration time 162.91 seconds +[May 1, 2026 - 03:24:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 260 : data loading time 0.00 +[May 1, 2026 - 03:24:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 260 : avg forward pass time 72.85 +[May 1, 2026 - 03:24:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 260 : backward pass time 8.42 +[May 1, 2026 - 03:24:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 260 : optimizer step time 0.39 +[May 1, 2026 - 03:24:40 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 260-------------------- +[May 1, 2026 - 03:24:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 243407.4404 iter count: 1.0 +[May 1, 2026 - 03:24:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6281 iter count: 1.0 +[May 1, 2026 - 03:24:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 243407.4404iter count: 1.0 +[May 1, 2026 - 03:24:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:24:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.4392iter count: 1.0 +[May 1, 2026 - 03:24:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.7010 iter count: 1.0 +[May 1, 2026 - 03:24:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.6708 iter count: 1.0 +[May 1, 2026 - 03:24:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 430371.2812iter count: 1.0 +[May 1, 2026 - 03:24:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 4.0634 iter count: 1.0 +[May 1, 2026 - 03:24:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 4599887.0000iter count: 1.0 +[May 1, 2026 - 03:24:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6281iter count: 1.0 +[May 1, 2026 - 03:24:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.7505 iter count: 1.0 +[May 1, 2026 - 03:24:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 98820.1328iter count: 1.0 +[May 1, 2026 - 03:24:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 0.7010 iter count: 1.0 +[May 1, 2026 - 03:24:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 4.0634 iter count: 1.0 +[May 1, 2026 - 03:24:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.1744 iter count: 1.0 +[May 1, 2026 - 03:24:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 571.7542iter count: 1.0 +[May 1, 2026 - 03:24:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.3996 iter count: 1.0 +[May 1, 2026 - 03:24:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 897.0849iter count: 1.0 +[May 1, 2026 - 03:24:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 1, 2026 - 03:24:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:24:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 03:25:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450089 41.606941 41.322273 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.686035 67.699219 67.646484 +util 94.000000 98.000000 90.000000 +[May 1, 2026 - 03:25:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 261 : avg iteration time 45.06 seconds +[May 1, 2026 - 03:25:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 261 : data loading time 0.00 +[May 1, 2026 - 03:25:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 261 : avg forward pass time 11.60 +[May 1, 2026 - 03:25:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 261 : backward pass time 10.13 +[May 1, 2026 - 03:25:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 261 : optimizer step time 1.57 +[May 1, 2026 - 03:25:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 261-------------------- +[May 1, 2026 - 03:25:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 03:25:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 03:25:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:25:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 03:26:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450337 41.607937 41.322273 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.681152 67.699219 67.626953 +util 93.000000 99.000000 86.000000 +[May 1, 2026 - 03:26:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 262 : avg iteration time 48.89 seconds +[May 1, 2026 - 03:26:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 262 : data loading time 0.00 +[May 1, 2026 - 03:26:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 262 : avg forward pass time 13.52 +[May 1, 2026 - 03:26:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 262 : backward pass time 10.15 +[May 1, 2026 - 03:26:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 262 : optimizer step time 1.56 +[May 1, 2026 - 03:26:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 262-------------------- +[May 1, 2026 - 03:26:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0104 iter count: 1.0 +[May 1, 2026 - 03:26:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 1, 2026 - 03:26:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:26:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 03:26:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450647 41.609177 41.322273 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.158691 67.529297 67.035156 +util 97.250000 99.000000 95.000000 +[May 1, 2026 - 03:26:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 263 : avg iteration time 43.56 seconds +[May 1, 2026 - 03:26:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 263 : data loading time 0.00 +[May 1, 2026 - 03:26:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 263 : avg forward pass time 10.87 +[May 1, 2026 - 03:26:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 263 : backward pass time 10.11 +[May 1, 2026 - 03:26:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 263 : optimizer step time 1.56 +[May 1, 2026 - 03:26:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 263-------------------- +[May 1, 2026 - 03:26:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 1, 2026 - 03:26:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 1, 2026 - 03:26:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:26:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 03:27:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450680 41.609295 41.322285 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.686035 67.699219 67.646484 +util 96.250000 100.000000 88.000000 +[May 1, 2026 - 03:27:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 264 : avg iteration time 46.23 seconds +[May 1, 2026 - 03:27:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 264 : data loading time 0.00 +[May 1, 2026 - 03:27:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 264 : avg forward pass time 12.19 +[May 1, 2026 - 03:27:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 264 : backward pass time 10.15 +[May 1, 2026 - 03:27:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 264 : optimizer step time 1.56 +[May 1, 2026 - 03:27:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 264-------------------- +[May 1, 2026 - 03:27:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 1, 2026 - 03:27:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 1, 2026 - 03:27:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:27:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 03:30:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450051 41.606808 41.322273 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.055664 136.076172 136.048828 +util 61.250000 66.000000 58.000000 +[May 1, 2026 - 03:30:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 265 : avg iteration time 162.90 seconds +[May 1, 2026 - 03:30:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 265 : data loading time 0.00 +[May 1, 2026 - 03:30:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 265 : avg forward pass time 72.82 +[May 1, 2026 - 03:30:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 265 : backward pass time 8.43 +[May 1, 2026 - 03:30:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 265 : optimizer step time 0.40 +[May 1, 2026 - 03:30:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 265-------------------- +[May 1, 2026 - 03:30:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 371863.1143 iter count: 1.0 +[May 1, 2026 - 03:30:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5958 iter count: 1.0 +[May 1, 2026 - 03:30:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 371863.1143iter count: 1.0 +[May 1, 2026 - 03:30:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:30:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.9707iter count: 1.0 +[May 1, 2026 - 03:30:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 2.5354 iter count: 1.0 +[May 1, 2026 - 03:30:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.2485 iter count: 1.0 +[May 1, 2026 - 03:30:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 694793.8125iter count: 1.0 +[May 1, 2026 - 03:30:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 159.3152iter count: 1.0 +[May 1, 2026 - 03:30:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 14605910.0000iter count: 1.0 +[May 1, 2026 - 03:30:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5958iter count: 1.0 +[May 1, 2026 - 03:30:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.8526 iter count: 1.0 +[May 1, 2026 - 03:30:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 121189.3672iter count: 1.0 +[May 1, 2026 - 03:30:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.2446 iter count: 1.0 +[May 1, 2026 - 03:30:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 658.0449iter count: 1.0 +[May 1, 2026 - 03:30:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.0919 iter count: 1.0 +[May 1, 2026 - 03:30:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 26471.0098iter count: 1.0 +[May 1, 2026 - 03:30:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.2174 iter count: 1.0 +[May 1, 2026 - 03:30:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 4604.3608iter count: 1.0 +[May 1, 2026 - 03:30:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 1, 2026 - 03:30:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:30:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 03:31:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450119 41.607071 41.322273 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.637207 67.699219 67.451172 +util 96.000000 99.000000 93.000000 +[May 1, 2026 - 03:31:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 266 : avg iteration time 47.84 seconds +[May 1, 2026 - 03:31:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 266 : data loading time 0.00 +[May 1, 2026 - 03:31:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 266 : avg forward pass time 13.00 +[May 1, 2026 - 03:31:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 266 : backward pass time 10.16 +[May 1, 2026 - 03:31:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 266 : optimizer step time 1.56 +[May 1, 2026 - 03:31:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 266-------------------- +[May 1, 2026 - 03:31:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0108 iter count: 1.0 +[May 1, 2026 - 03:31:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 1, 2026 - 03:31:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:31:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 03:32:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450499 41.608551 41.322273 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.471191 67.484375 67.431641 +util 92.000000 96.000000 88.000000 +[May 1, 2026 - 03:32:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 267 : avg iteration time 46.34 seconds +[May 1, 2026 - 03:32:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 267 : data loading time 0.00 +[May 1, 2026 - 03:32:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 267 : avg forward pass time 12.22 +[May 1, 2026 - 03:32:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 267 : backward pass time 10.15 +[May 1, 2026 - 03:32:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 267 : optimizer step time 1.58 +[May 1, 2026 - 03:32:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 267-------------------- +[May 1, 2026 - 03:32:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 03:32:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 03:32:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:32:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 03:32:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450559 41.608791 41.322273 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 69.028809 69.457031 67.744141 +util 92.000000 96.000000 88.000000 +[May 1, 2026 - 03:32:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 268 : avg iteration time 49.00 seconds +[May 1, 2026 - 03:32:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 268 : data loading time 0.00 +[May 1, 2026 - 03:32:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 268 : avg forward pass time 13.57 +[May 1, 2026 - 03:32:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 268 : backward pass time 10.15 +[May 1, 2026 - 03:32:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 268 : optimizer step time 1.57 +[May 1, 2026 - 03:32:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 268-------------------- +[May 1, 2026 - 03:32:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0086 iter count: 1.0 +[May 1, 2026 - 03:32:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 1, 2026 - 03:32:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:32:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 03:33:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450618 41.609066 41.322273 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.965332 69.476562 67.431641 +util 94.000000 98.000000 90.000000 +[May 1, 2026 - 03:33:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 269 : avg iteration time 43.62 seconds +[May 1, 2026 - 03:33:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 269 : data loading time 0.00 +[May 1, 2026 - 03:33:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 269 : avg forward pass time 10.87 +[May 1, 2026 - 03:33:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 269 : backward pass time 10.13 +[May 1, 2026 - 03:33:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 269 : optimizer step time 1.58 +[May 1, 2026 - 03:33:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 269-------------------- +[May 1, 2026 - 03:33:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 1, 2026 - 03:33:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 1, 2026 - 03:33:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:33:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 03:36:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450390 41.607208 41.322273 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.075195 136.095703 136.068359 +util 61.750000 68.000000 55.000000 +[May 1, 2026 - 03:36:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 270 : avg iteration time 163.48 seconds +[May 1, 2026 - 03:36:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 270 : data loading time 0.00 +[May 1, 2026 - 03:36:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 270 : avg forward pass time 73.16 +[May 1, 2026 - 03:36:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 270 : backward pass time 8.40 +[May 1, 2026 - 03:36:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 270 : optimizer step time 0.40 +[May 1, 2026 - 03:36:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 270-------------------- +[May 1, 2026 - 03:36:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 575246.9927 iter count: 1.0 +[May 1, 2026 - 03:36:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6457 iter count: 1.0 +[May 1, 2026 - 03:36:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 575246.9927iter count: 1.0 +[May 1, 2026 - 03:36:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:36:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.1847iter count: 1.0 +[May 1, 2026 - 03:36:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.6632 iter count: 1.0 +[May 1, 2026 - 03:36:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.6200 iter count: 1.0 +[May 1, 2026 - 03:36:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 1150426.2500iter count: 1.0 +[May 1, 2026 - 03:36:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 3.7673 iter count: 1.0 +[May 1, 2026 - 03:36:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 30705610.0000iter count: 1.0 +[May 1, 2026 - 03:36:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6457iter count: 1.0 +[May 1, 2026 - 03:36:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.3515 iter count: 1.0 +[May 1, 2026 - 03:36:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 814.8802iter count: 1.0 +[May 1, 2026 - 03:36:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.7823 iter count: 1.0 +[May 1, 2026 - 03:36:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 105293.9609iter count: 1.0 +[May 1, 2026 - 03:36:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.0713 iter count: 1.0 +[May 1, 2026 - 03:36:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 3438.0188iter count: 1.0 +[May 1, 2026 - 03:36:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.6924 iter count: 1.0 +[May 1, 2026 - 03:36:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 11904.8906iter count: 1.0 +[May 1, 2026 - 03:36:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[May 1, 2026 - 03:36:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:36:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 03:37:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450453 41.607445 41.322273 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.471191 67.484375 67.431641 +util 95.250000 99.000000 92.000000 +[May 1, 2026 - 03:37:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 271 : avg iteration time 50.51 seconds +[May 1, 2026 - 03:37:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 271 : data loading time 0.00 +[May 1, 2026 - 03:37:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 271 : avg forward pass time 14.34 +[May 1, 2026 - 03:37:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 271 : backward pass time 10.12 +[May 1, 2026 - 03:37:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 271 : optimizer step time 1.56 +[May 1, 2026 - 03:37:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 271-------------------- +[May 1, 2026 - 03:37:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0101 iter count: 1.0 +[May 1, 2026 - 03:37:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 1, 2026 - 03:37:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:37:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 03:37:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450825 41.608948 41.322273 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.417480 67.509766 67.386719 +util 96.250000 100.000000 91.000000 +[May 1, 2026 - 03:37:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 272 : avg iteration time 43.53 seconds +[May 1, 2026 - 03:37:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 272 : data loading time 0.00 +[May 1, 2026 - 03:37:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 272 : avg forward pass time 10.83 +[May 1, 2026 - 03:37:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 272 : backward pass time 10.14 +[May 1, 2026 - 03:37:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 272 : optimizer step time 1.58 +[May 1, 2026 - 03:37:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 272-------------------- +[May 1, 2026 - 03:37:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 03:37:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 03:37:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:37:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 03:38:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450863 41.609100 41.322273 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.515137 67.542969 67.431641 +util 93.000000 97.000000 89.000000 +[May 1, 2026 - 03:38:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 273 : avg iteration time 49.06 seconds +[May 1, 2026 - 03:38:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 273 : data loading time 0.00 +[May 1, 2026 - 03:38:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 273 : avg forward pass time 13.57 +[May 1, 2026 - 03:38:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 273 : backward pass time 10.16 +[May 1, 2026 - 03:38:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 273 : optimizer step time 1.59 +[May 1, 2026 - 03:38:40 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 273-------------------- +[May 1, 2026 - 03:38:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 1, 2026 - 03:38:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 1, 2026 - 03:38:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:38:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 03:39:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450722 41.608536 41.322273 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 69.214355 69.808594 67.431641 +util 96.000000 99.000000 93.000000 +[May 1, 2026 - 03:39:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 274 : avg iteration time 43.62 seconds +[May 1, 2026 - 03:39:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 274 : data loading time 0.00 +[May 1, 2026 - 03:39:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 274 : avg forward pass time 10.89 +[May 1, 2026 - 03:39:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 274 : backward pass time 10.13 +[May 1, 2026 - 03:39:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 274 : optimizer step time 1.56 +[May 1, 2026 - 03:39:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 274-------------------- +[May 1, 2026 - 03:39:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 1, 2026 - 03:39:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 1, 2026 - 03:39:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:39:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 03:42:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450640 41.608158 41.322273 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.026367 136.029297 136.017578 +util 64.750000 68.000000 62.000000 +[May 1, 2026 - 03:42:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 275 : avg iteration time 166.00 seconds +[May 1, 2026 - 03:42:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 275 : data loading time 0.00 +[May 1, 2026 - 03:42:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 275 : avg forward pass time 74.39 +[May 1, 2026 - 03:42:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 275 : backward pass time 8.41 +[May 1, 2026 - 03:42:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 275 : optimizer step time 0.39 +[May 1, 2026 - 03:42:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 275-------------------- +[May 1, 2026 - 03:42:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 9924842.4824 iter count: 1.0 +[May 1, 2026 - 03:42:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6293 iter count: 1.0 +[May 1, 2026 - 03:42:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 9924842.4824iter count: 1.0 +[May 1, 2026 - 03:42:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:42:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.9684iter count: 1.0 +[May 1, 2026 - 03:42:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.8764 iter count: 1.0 +[May 1, 2026 - 03:42:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 9.8636 iter count: 1.0 +[May 1, 2026 - 03:42:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 11818257.0000iter count: 1.0 +[May 1, 2026 - 03:42:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 5.7708 iter count: 1.0 +[May 1, 2026 - 03:42:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 369301248.0000iter count: 1.0 +[May 1, 2026 - 03:42:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6293iter count: 1.0 +[May 1, 2026 - 03:42:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.4964 iter count: 1.0 +[May 1, 2026 - 03:42:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 8045.6704iter count: 1.0 +[May 1, 2026 - 03:42:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.5519 iter count: 1.0 +[May 1, 2026 - 03:42:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 8989.7119iter count: 1.0 +[May 1, 2026 - 03:42:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.5226 iter count: 1.0 +[May 1, 2026 - 03:42:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 62645.0781iter count: 1.0 +[May 1, 2026 - 03:42:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.7017 iter count: 1.0 +[May 1, 2026 - 03:42:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 662256.3750iter count: 1.0 +[May 1, 2026 - 03:42:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[May 1, 2026 - 03:42:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:42:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 03:43:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.450704 41.608402 41.322285 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.266113 67.490234 67.191406 +util 95.250000 100.000000 90.000000 +[May 1, 2026 - 03:43:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 276 : avg iteration time 50.70 seconds +[May 1, 2026 - 03:43:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 276 : data loading time 0.00 +[May 1, 2026 - 03:43:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 276 : avg forward pass time 14.42 +[May 1, 2026 - 03:43:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 276 : backward pass time 10.12 +[May 1, 2026 - 03:43:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 276 : optimizer step time 1.57 +[May 1, 2026 - 03:43:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 276-------------------- +[May 1, 2026 - 03:43:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0104 iter count: 1.0 +[May 1, 2026 - 03:43:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 1, 2026 - 03:43:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:43:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 03:43:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.451079 41.609901 41.322285 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.676270 67.757812 67.431641 +util 98.750000 100.000000 97.000000 +[May 1, 2026 - 03:43:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 277 : avg iteration time 48.94 seconds +[May 1, 2026 - 03:43:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 277 : data loading time 0.00 +[May 1, 2026 - 03:43:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 277 : avg forward pass time 13.55 +[May 1, 2026 - 03:43:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 277 : backward pass time 10.13 +[May 1, 2026 - 03:43:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 277 : optimizer step time 1.58 +[May 1, 2026 - 03:43:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 277-------------------- +[May 1, 2026 - 03:43:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 1, 2026 - 03:43:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 1, 2026 - 03:43:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:43:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 03:44:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.451150 41.610172 41.322285 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.788574 67.914062 67.412109 +util 93.500000 99.000000 87.000000 +[May 1, 2026 - 03:44:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 278 : avg iteration time 49.02 seconds +[May 1, 2026 - 03:44:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 278 : data loading time 0.00 +[May 1, 2026 - 03:44:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 278 : avg forward pass time 13.57 +[May 1, 2026 - 03:44:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 278 : backward pass time 10.14 +[May 1, 2026 - 03:44:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 278 : optimizer step time 1.57 +[May 1, 2026 - 03:44:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 278-------------------- +[May 1, 2026 - 03:44:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 03:44:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 03:44:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:44:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 03:45:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.451035 41.609711 41.322285 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.837402 67.953125 67.490234 +util 94.250000 98.000000 91.000000 +[May 1, 2026 - 03:45:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 279 : avg iteration time 43.51 seconds +[May 1, 2026 - 03:45:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 279 : data loading time 0.00 +[May 1, 2026 - 03:45:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 279 : avg forward pass time 10.83 +[May 1, 2026 - 03:45:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 279 : backward pass time 10.13 +[May 1, 2026 - 03:45:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 279 : optimizer step time 1.58 +[May 1, 2026 - 03:45:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 279-------------------- +[May 1, 2026 - 03:45:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 1, 2026 - 03:45:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 1, 2026 - 03:45:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:45:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 03:48:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.451112 41.608452 41.322510 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.031250 136.154297 135.990234 +util 67.750000 73.000000 61.000000 +[May 1, 2026 - 03:48:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 280 : avg iteration time 168.96 seconds +[May 1, 2026 - 03:48:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 280 : data loading time 0.00 +[May 1, 2026 - 03:48:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 280 : avg forward pass time 75.88 +[May 1, 2026 - 03:48:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 280 : backward pass time 8.39 +[May 1, 2026 - 03:48:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 280 : optimizer step time 0.40 +[May 1, 2026 - 03:48:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 280-------------------- +[May 1, 2026 - 03:48:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 239873.3486 iter count: 1.0 +[May 1, 2026 - 03:48:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6108 iter count: 1.0 +[May 1, 2026 - 03:48:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 239873.3486iter count: 1.0 +[May 1, 2026 - 03:48:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:48:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 5.1210iter count: 1.0 +[May 1, 2026 - 03:48:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 2.5569 iter count: 1.0 +[May 1, 2026 - 03:48:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.8211 iter count: 1.0 +[May 1, 2026 - 03:48:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 338295.3750iter count: 1.0 +[May 1, 2026 - 03:48:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 166.3170iter count: 1.0 +[May 1, 2026 - 03:48:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 6212776.5000iter count: 1.0 +[May 1, 2026 - 03:48:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6108iter count: 1.0 +[May 1, 2026 - 03:48:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.3162 iter count: 1.0 +[May 1, 2026 - 03:48:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 41453.7578iter count: 1.0 +[May 1, 2026 - 03:48:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.4931 iter count: 1.0 +[May 1, 2026 - 03:48:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 59048.6211iter count: 1.0 +[May 1, 2026 - 03:48:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 6.6722 iter count: 1.0 +[May 1, 2026 - 03:48:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 624258.8125iter count: 1.0 +[May 1, 2026 - 03:48:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.5061 iter count: 1.0 +[May 1, 2026 - 03:48:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 8202.3389iter count: 1.0 +[May 1, 2026 - 03:48:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 1, 2026 - 03:48:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:48:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 03:48:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.451112 41.608444 41.322510 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.612793 67.660156 67.470703 +util 93.250000 99.000000 87.000000 +[May 1, 2026 - 03:48:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 281 : avg iteration time 45.13 seconds +[May 1, 2026 - 03:48:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 281 : data loading time 0.00 +[May 1, 2026 - 03:48:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 281 : avg forward pass time 11.63 +[May 1, 2026 - 03:48:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 281 : backward pass time 10.15 +[May 1, 2026 - 03:48:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 281 : optimizer step time 1.58 +[May 1, 2026 - 03:48:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 281-------------------- +[May 1, 2026 - 03:48:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 03:48:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 03:48:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:48:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 03:49:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.451267 41.608940 41.322521 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.490723 67.509766 67.484375 +util 96.250000 99.000000 93.000000 +[May 1, 2026 - 03:49:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 282 : avg iteration time 48.94 seconds +[May 1, 2026 - 03:49:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 282 : data loading time 0.00 +[May 1, 2026 - 03:49:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 282 : avg forward pass time 13.55 +[May 1, 2026 - 03:49:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 282 : backward pass time 10.13 +[May 1, 2026 - 03:49:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 282 : optimizer step time 1.57 +[May 1, 2026 - 03:49:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 282-------------------- +[May 1, 2026 - 03:49:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 1, 2026 - 03:49:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 1, 2026 - 03:49:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:49:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 03:50:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.451261 41.608952 41.322517 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.676270 67.757812 67.431641 +util 92.000000 96.000000 89.000000 +[May 1, 2026 - 03:50:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 283 : avg iteration time 48.94 seconds +[May 1, 2026 - 03:50:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 283 : data loading time 0.00 +[May 1, 2026 - 03:50:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 283 : avg forward pass time 13.54 +[May 1, 2026 - 03:50:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 283 : backward pass time 10.14 +[May 1, 2026 - 03:50:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 283 : optimizer step time 1.56 +[May 1, 2026 - 03:50:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 283-------------------- +[May 1, 2026 - 03:50:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 03:50:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 03:50:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:50:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 03:51:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.451258 41.608940 41.322517 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.510254 67.523438 67.470703 +util 97.750000 100.000000 95.000000 +[May 1, 2026 - 03:51:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 284 : avg iteration time 46.28 seconds +[May 1, 2026 - 03:51:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 284 : data loading time 0.00 +[May 1, 2026 - 03:51:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 284 : avg forward pass time 12.21 +[May 1, 2026 - 03:51:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 284 : backward pass time 10.14 +[May 1, 2026 - 03:51:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 284 : optimizer step time 1.58 +[May 1, 2026 - 03:51:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 284-------------------- +[May 1, 2026 - 03:51:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 03:51:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 03:51:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:51:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 03:54:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.451090 41.608253 41.322517 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.080078 136.087891 136.056641 +util 67.500000 72.000000 64.000000 +[May 1, 2026 - 03:54:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 285 : avg iteration time 166.02 seconds +[May 1, 2026 - 03:54:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 285 : data loading time 0.00 +[May 1, 2026 - 03:54:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 285 : avg forward pass time 74.37 +[May 1, 2026 - 03:54:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 285 : backward pass time 8.44 +[May 1, 2026 - 03:54:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 285 : optimizer step time 0.41 +[May 1, 2026 - 03:54:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 285-------------------- +[May 1, 2026 - 03:54:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 237559.9189 iter count: 1.0 +[May 1, 2026 - 03:54:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6583 iter count: 1.0 +[May 1, 2026 - 03:54:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 237559.9189iter count: 1.0 +[May 1, 2026 - 03:54:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:54:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.8900iter count: 1.0 +[May 1, 2026 - 03:54:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 2.6901 iter count: 1.0 +[May 1, 2026 - 03:54:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.3648 iter count: 1.0 +[May 1, 2026 - 03:54:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 335742.6250iter count: 1.0 +[May 1, 2026 - 03:54:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 217.0444iter count: 1.0 +[May 1, 2026 - 03:54:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 2494350.5000iter count: 1.0 +[May 1, 2026 - 03:54:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6583iter count: 1.0 +[May 1, 2026 - 03:54:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.9454 iter count: 1.0 +[May 1, 2026 - 03:54:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 145919.1719iter count: 1.0 +[May 1, 2026 - 03:54:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 7.0186 iter count: 1.0 +[May 1, 2026 - 03:54:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 1248193.2500iter count: 1.0 +[May 1, 2026 - 03:54:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.4282 iter count: 1.0 +[May 1, 2026 - 03:54:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 949.9261iter count: 1.0 +[May 1, 2026 - 03:54:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.5261 iter count: 1.0 +[May 1, 2026 - 03:54:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 8537.4814iter count: 1.0 +[May 1, 2026 - 03:54:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 1, 2026 - 03:54:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:54:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 03:54:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.451091 41.608253 41.322517 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 69.253418 69.828125 67.529297 +util 98.250000 100.000000 97.000000 +[May 1, 2026 - 03:54:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 286 : avg iteration time 45.39 seconds +[May 1, 2026 - 03:54:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 286 : data loading time 0.00 +[May 1, 2026 - 03:54:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 286 : avg forward pass time 11.76 +[May 1, 2026 - 03:54:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 286 : backward pass time 10.15 +[May 1, 2026 - 03:54:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 286 : optimizer step time 1.58 +[May 1, 2026 - 03:54:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 286-------------------- +[May 1, 2026 - 03:54:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 03:54:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 03:54:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:54:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 03:55:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.451531 41.609985 41.322525 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.598145 67.660156 67.412109 +util 97.500000 100.000000 95.000000 +[May 1, 2026 - 03:55:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 287 : avg iteration time 49.18 seconds +[May 1, 2026 - 03:55:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 287 : data loading time 0.00 +[May 1, 2026 - 03:55:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 287 : avg forward pass time 13.67 +[May 1, 2026 - 03:55:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 287 : backward pass time 10.13 +[May 1, 2026 - 03:55:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 287 : optimizer step time 1.56 +[May 1, 2026 - 03:55:40 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 287-------------------- +[May 1, 2026 - 03:55:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 03:55:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 03:55:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:55:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 03:56:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.451595 41.610237 41.322517 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.192871 67.431641 67.113281 +util 96.250000 99.000000 93.000000 +[May 1, 2026 - 03:56:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 288 : avg iteration time 49.18 seconds +[May 1, 2026 - 03:56:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 288 : data loading time 0.00 +[May 1, 2026 - 03:56:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 288 : avg forward pass time 13.67 +[May 1, 2026 - 03:56:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 288 : backward pass time 10.12 +[May 1, 2026 - 03:56:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 288 : optimizer step time 1.59 +[May 1, 2026 - 03:56:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 288-------------------- +[May 1, 2026 - 03:56:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 03:56:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 03:56:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:56:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 03:57:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.451402 41.609497 41.322517 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.887207 69.378906 67.412109 +util 94.750000 100.000000 89.000000 +[May 1, 2026 - 03:57:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 289 : avg iteration time 48.89 seconds +[May 1, 2026 - 03:57:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 289 : data loading time 0.00 +[May 1, 2026 - 03:57:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 289 : avg forward pass time 13.53 +[May 1, 2026 - 03:57:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 289 : backward pass time 10.14 +[May 1, 2026 - 03:57:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 289 : optimizer step time 1.57 +[May 1, 2026 - 03:57:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 289-------------------- +[May 1, 2026 - 03:57:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 1, 2026 - 03:57:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 1, 2026 - 03:57:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 03:57:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 04:00:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.451452 41.608723 41.322517 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.066895 136.070312 136.056641 +util 65.000000 73.000000 61.000000 +[May 1, 2026 - 04:00:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 290 : avg iteration time 168.63 seconds +[May 1, 2026 - 04:00:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 290 : data loading time 0.00 +[May 1, 2026 - 04:00:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 290 : avg forward pass time 75.70 +[May 1, 2026 - 04:00:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 290 : backward pass time 8.44 +[May 1, 2026 - 04:00:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 290 : optimizer step time 0.39 +[May 1, 2026 - 04:00:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 290-------------------- +[May 1, 2026 - 04:00:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 499234.6084 iter count: 1.0 +[May 1, 2026 - 04:00:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5931 iter count: 1.0 +[May 1, 2026 - 04:00:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 499234.6084iter count: 1.0 +[May 1, 2026 - 04:00:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:00:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.9256iter count: 1.0 +[May 1, 2026 - 04:00:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.7724 iter count: 1.0 +[May 1, 2026 - 04:00:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.6199 iter count: 1.0 +[May 1, 2026 - 04:00:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 1072939.1250iter count: 1.0 +[May 1, 2026 - 04:00:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 34.6304iter count: 1.0 +[May 1, 2026 - 04:00:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 30699050.0000iter count: 1.0 +[May 1, 2026 - 04:00:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5931iter count: 1.0 +[May 1, 2026 - 04:00:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.5531 iter count: 1.0 +[May 1, 2026 - 04:00:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 1219.5038iter count: 1.0 +[May 1, 2026 - 04:00:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 1.7724 iter count: 1.0 +[May 1, 2026 - 04:00:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 34.6304 iter count: 1.0 +[May 1, 2026 - 04:00:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.8266 iter count: 1.0 +[May 1, 2026 - 04:00:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 2107.5718iter count: 1.0 +[May 1, 2026 - 04:00:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.5883 iter count: 1.0 +[May 1, 2026 - 04:00:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 9668.1279iter count: 1.0 +[May 1, 2026 - 04:00:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 04:00:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:00:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 04:00:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.451448 41.608723 41.322517 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.236816 67.490234 67.152344 +util 94.500000 100.000000 88.000000 +[May 1, 2026 - 04:00:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 291 : avg iteration time 45.15 seconds +[May 1, 2026 - 04:00:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 291 : data loading time 0.00 +[May 1, 2026 - 04:00:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 291 : avg forward pass time 11.65 +[May 1, 2026 - 04:00:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 291 : backward pass time 10.15 +[May 1, 2026 - 04:00:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 291 : optimizer step time 1.57 +[May 1, 2026 - 04:00:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 291-------------------- +[May 1, 2026 - 04:00:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 04:00:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 04:00:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:00:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 04:01:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.451517 41.608974 41.322540 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.427246 67.548828 67.386719 +util 94.250000 100.000000 88.000000 +[May 1, 2026 - 04:01:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 292 : avg iteration time 48.99 seconds +[May 1, 2026 - 04:01:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 292 : data loading time 0.00 +[May 1, 2026 - 04:01:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 292 : avg forward pass time 13.57 +[May 1, 2026 - 04:01:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 292 : backward pass time 10.13 +[May 1, 2026 - 04:01:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 292 : optimizer step time 1.56 +[May 1, 2026 - 04:01:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 292-------------------- +[May 1, 2026 - 04:01:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 04:01:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 04:01:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:01:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 04:02:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.451576 41.609222 41.322521 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.427246 67.431641 67.425781 +util 95.750000 100.000000 88.000000 +[May 1, 2026 - 04:02:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 293 : avg iteration time 46.37 seconds +[May 1, 2026 - 04:02:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 293 : data loading time 0.00 +[May 1, 2026 - 04:02:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 293 : avg forward pass time 12.25 +[May 1, 2026 - 04:02:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 293 : backward pass time 10.14 +[May 1, 2026 - 04:02:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 293 : optimizer step time 1.57 +[May 1, 2026 - 04:02:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 293-------------------- +[May 1, 2026 - 04:02:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 04:02:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 04:02:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:02:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 04:03:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.451613 41.609222 41.322639 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.461426 67.626953 67.406250 +util 94.750000 100.000000 89.000000 +[May 1, 2026 - 04:03:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 294 : avg iteration time 46.23 seconds +[May 1, 2026 - 04:03:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 294 : data loading time 0.00 +[May 1, 2026 - 04:03:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 294 : avg forward pass time 12.18 +[May 1, 2026 - 04:03:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 294 : backward pass time 10.13 +[May 1, 2026 - 04:03:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 294 : optimizer step time 1.57 +[May 1, 2026 - 04:03:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 294-------------------- +[May 1, 2026 - 04:03:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0086 iter count: 1.0 +[May 1, 2026 - 04:03:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 1, 2026 - 04:03:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:03:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 04:06:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.451394 41.608360 41.322639 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.084961 136.087891 136.076172 +util 66.250000 72.000000 59.000000 +[May 1, 2026 - 04:06:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 295 : avg iteration time 168.64 seconds +[May 1, 2026 - 04:06:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 295 : data loading time 0.00 +[May 1, 2026 - 04:06:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 295 : avg forward pass time 75.72 +[May 1, 2026 - 04:06:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 295 : backward pass time 8.39 +[May 1, 2026 - 04:06:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 295 : optimizer step time 0.41 +[May 1, 2026 - 04:06:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 295-------------------- +[May 1, 2026 - 04:06:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 168882.4932 iter count: 1.0 +[May 1, 2026 - 04:06:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6030 iter count: 1.0 +[May 1, 2026 - 04:06:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 168882.4932iter count: 1.0 +[May 1, 2026 - 04:06:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:06:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.4118iter count: 1.0 +[May 1, 2026 - 04:06:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.9804 iter count: 1.0 +[May 1, 2026 - 04:06:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.8269 iter count: 1.0 +[May 1, 2026 - 04:06:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 300423.8125iter count: 1.0 +[May 1, 2026 - 04:06:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 7.1055 iter count: 1.0 +[May 1, 2026 - 04:06:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 6286238.5000iter count: 1.0 +[May 1, 2026 - 04:06:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6030iter count: 1.0 +[May 1, 2026 - 04:06:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.8182 iter count: 1.0 +[May 1, 2026 - 04:06:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 15312.4160iter count: 1.0 +[May 1, 2026 - 04:06:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.5362 iter count: 1.0 +[May 1, 2026 - 04:06:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 8711.1338iter count: 1.0 +[May 1, 2026 - 04:06:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 2.6920 iter count: 1.0 +[May 1, 2026 - 04:06:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 217.9108iter count: 1.0 +[May 1, 2026 - 04:06:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.7283 iter count: 1.0 +[May 1, 2026 - 04:06:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 12792.1963iter count: 1.0 +[May 1, 2026 - 04:06:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 1, 2026 - 04:06:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:06:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 04:06:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.451397 41.608360 41.322639 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.612793 67.679688 67.412109 +util 94.750000 100.000000 88.000000 +[May 1, 2026 - 04:06:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 296 : avg iteration time 45.16 seconds +[May 1, 2026 - 04:06:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 296 : data loading time 0.00 +[May 1, 2026 - 04:06:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 296 : avg forward pass time 11.66 +[May 1, 2026 - 04:06:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 296 : backward pass time 10.13 +[May 1, 2026 - 04:06:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 296 : optimizer step time 1.56 +[May 1, 2026 - 04:06:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 296-------------------- +[May 1, 2026 - 04:06:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 04:06:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 04:06:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:06:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 04:07:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.451650 41.609367 41.322643 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.627441 67.660156 67.529297 +util 94.750000 100.000000 89.000000 +[May 1, 2026 - 04:07:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 297 : avg iteration time 43.61 seconds +[May 1, 2026 - 04:07:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 297 : data loading time 0.00 +[May 1, 2026 - 04:07:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 297 : avg forward pass time 10.85 +[May 1, 2026 - 04:07:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 297 : backward pass time 10.18 +[May 1, 2026 - 04:07:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 297 : optimizer step time 1.58 +[May 1, 2026 - 04:07:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 297-------------------- +[May 1, 2026 - 04:07:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 04:07:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 04:07:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:07:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 04:08:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.451658 41.609367 41.322674 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.227051 67.451172 67.152344 +util 98.000000 99.000000 97.000000 +[May 1, 2026 - 04:08:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 298 : avg iteration time 48.95 seconds +[May 1, 2026 - 04:08:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 298 : data loading time 0.00 +[May 1, 2026 - 04:08:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 298 : avg forward pass time 13.54 +[May 1, 2026 - 04:08:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 298 : backward pass time 10.14 +[May 1, 2026 - 04:08:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 298 : optimizer step time 1.57 +[May 1, 2026 - 04:08:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 298-------------------- +[May 1, 2026 - 04:08:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 1, 2026 - 04:08:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 1, 2026 - 04:08:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:08:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 04:09:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.451701 41.609367 41.322685 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.578613 67.640625 67.392578 +util 95.750000 100.000000 91.000000 +[May 1, 2026 - 04:09:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 299 : avg iteration time 46.27 seconds +[May 1, 2026 - 04:09:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 299 : data loading time 0.00 +[May 1, 2026 - 04:09:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 299 : avg forward pass time 12.21 +[May 1, 2026 - 04:09:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 299 : backward pass time 10.13 +[May 1, 2026 - 04:09:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 299 : optimizer step time 1.58 +[May 1, 2026 - 04:09:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 299-------------------- +[May 1, 2026 - 04:09:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 04:09:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 04:09:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:09:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 04:11:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.451582 41.608116 41.322662 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.110840 136.115234 136.109375 +util 68.500000 73.000000 66.000000 +[May 1, 2026 - 04:11:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 300 : avg iteration time 166.13 seconds +[May 1, 2026 - 04:11:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 300 : data loading time 0.00 +[May 1, 2026 - 04:11:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 300 : avg forward pass time 74.47 +[May 1, 2026 - 04:11:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 300 : backward pass time 8.42 +[May 1, 2026 - 04:11:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 300 : optimizer step time 0.39 +[May 1, 2026 - 04:11:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 300-------------------- +[May 1, 2026 - 04:11:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 840845.8906 iter count: 1.0 +[May 1, 2026 - 04:11:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6329 iter count: 1.0 +[May 1, 2026 - 04:11:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 840845.8906iter count: 1.0 +[May 1, 2026 - 04:11:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:11:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.8872iter count: 1.0 +[May 1, 2026 - 04:11:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.5217 iter count: 1.0 +[May 1, 2026 - 04:11:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.3420 iter count: 1.0 +[May 1, 2026 - 04:11:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 1553317.0000iter count: 1.0 +[May 1, 2026 - 04:11:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 2.8387 iter count: 1.0 +[May 1, 2026 - 04:11:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 17610096.0000iter count: 1.0 +[May 1, 2026 - 04:11:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6329iter count: 1.0 +[May 1, 2026 - 04:11:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 8.1744 iter count: 1.0 +[May 1, 2026 - 04:11:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 12594828.0000iter count: 1.0 +[May 1, 2026 - 04:11:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 6.6639 iter count: 1.0 +[May 1, 2026 - 04:11:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 614057.3750iter count: 1.0 +[May 1, 2026 - 04:11:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 7.0612 iter count: 1.0 +[May 1, 2026 - 04:11:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 1359306.3750iter count: 1.0 +[May 1, 2026 - 04:11:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 8.0029 iter count: 1.0 +[May 1, 2026 - 04:11:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 8937956.0000iter count: 1.0 +[May 1, 2026 - 04:11:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 04:11:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 1, 2026 - 04:12:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 14.88 seconds +[May 1, 2026 - 04:12:08 | INFO | fastgen.trainer:save_checkpoint:266 ] Saving checkpoint iteration 300 +[May 1, 2026 - 04:12:08 | INFO | fastgen.utils.checkpointer:save:311 ] Saving FSDP model to prefix /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000300 +[May 1, 2026 - 04:12:41 | SUCCESS | fastgen.utils.checkpointer:save:362 ] Model saved at iteration 300 +[May 1, 2026 - 04:12:42 | INFO | fastgen.trainer:validate:358 ] Validation iteration 300 +[May 1, 2026 - 04:12:42 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 04:12:44 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 2.1s +[May 1, 2026 - 04:12:47 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 0: sync_c=6.460 +[May 1, 2026 - 04:12:48 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 04:12:49 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 04:12:52 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 1: sync_c=6.134 +[May 1, 2026 - 04:12:53 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 04:12:54 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 04:12:57 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 2: sync_c=5.482 +[May 1, 2026 - 04:12:58 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 04:12:59 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 04:13:02 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 3: sync_c=4.750 +[May 1, 2026 - 04:13:03 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 04:13:05 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 04:13:08 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 4: sync_c=5.226 +[May 1, 2026 - 04:13:08 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 04:13:10 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 04:13:13 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 5: sync_c=6.938 +[May 1, 2026 - 04:13:13 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 04:13:15 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 04:13:18 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 6: sync_c=6.892 +[May 1, 2026 - 04:13:18 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 04:13:20 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 04:13:23 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 7: sync_c=4.731 +[May 1, 2026 - 04:13:23 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 04:13:25 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 04:13:28 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 8: sync_c=6.277 +[May 1, 2026 - 04:13:28 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 04:13:30 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 04:13:33 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 9: sync_c=3.562 +[May 1, 2026 - 04:13:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging val0 stats at iteration 300-------------------- +[May 1, 2026 - 04:13:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0000 iter count: 10.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 1, 2026 - 04:13:50 | INFO | fastgen.callbacks.wandb:on_validation_end:763 ] Logged 10 val videos at iteration 300 +[May 1, 2026 - 04:13:50 | INFO | fastgen.callbacks.wandb:on_validation_end:769 ] [SyncEval] val0 mean sync_c: 5.645 (n=10) +[May 1, 2026 - 04:14:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452895 41.613670 41.322578 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.433105 68.876953 68.285156 +util 92.000000 96.000000 89.000000 +[May 1, 2026 - 04:14:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 301 : avg iteration time 162.71 seconds +[May 1, 2026 - 04:14:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 301 : data loading time 0.00 +[May 1, 2026 - 04:14:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 301 : avg forward pass time 11.85 +[May 1, 2026 - 04:14:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 301 : backward pass time 10.16 +[May 1, 2026 - 04:14:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 301 : optimizer step time 1.55 +[May 1, 2026 - 04:14:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 301-------------------- +[May 1, 2026 - 04:14:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 04:14:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 04:14:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:14:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 04:15:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452887 41.613670 41.322567 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.389160 68.876953 68.226562 +util 95.750000 100.000000 88.000000 +[May 1, 2026 - 04:15:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 302 : avg iteration time 46.51 seconds +[May 1, 2026 - 04:15:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 302 : data loading time 0.00 +[May 1, 2026 - 04:15:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 302 : avg forward pass time 12.29 +[May 1, 2026 - 04:15:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 302 : backward pass time 10.18 +[May 1, 2026 - 04:15:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 302 : optimizer step time 1.57 +[May 1, 2026 - 04:15:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 302-------------------- +[May 1, 2026 - 04:15:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 04:15:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 04:15:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:15:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 04:16:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452902 41.613670 41.322605 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.369629 68.740234 68.246094 +util 97.500000 100.000000 95.000000 +[May 1, 2026 - 04:16:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 303 : avg iteration time 46.57 seconds +[May 1, 2026 - 04:16:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 303 : data loading time 0.00 +[May 1, 2026 - 04:16:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 303 : avg forward pass time 12.30 +[May 1, 2026 - 04:16:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 303 : backward pass time 10.19 +[May 1, 2026 - 04:16:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 303 : optimizer step time 1.58 +[May 1, 2026 - 04:16:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 303-------------------- +[May 1, 2026 - 04:16:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 04:16:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 04:16:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:16:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 04:16:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452719 41.612938 41.322594 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.369629 68.740234 68.246094 +util 92.750000 97.000000 89.000000 +[May 1, 2026 - 04:16:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 304 : avg iteration time 49.13 seconds +[May 1, 2026 - 04:16:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 304 : data loading time 0.00 +[May 1, 2026 - 04:16:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 304 : avg forward pass time 13.61 +[May 1, 2026 - 04:16:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 304 : backward pass time 10.17 +[May 1, 2026 - 04:16:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 304 : optimizer step time 1.57 +[May 1, 2026 - 04:16:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 304-------------------- +[May 1, 2026 - 04:16:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 04:16:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 04:16:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:16:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 04:19:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452560 41.610832 41.323044 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.027832 136.031250 136.017578 +util 66.000000 74.000000 62.000000 +[May 1, 2026 - 04:19:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 305 : avg iteration time 171.58 seconds +[May 1, 2026 - 04:19:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 305 : data loading time 0.00 +[May 1, 2026 - 04:19:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 305 : avg forward pass time 77.21 +[May 1, 2026 - 04:19:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 305 : backward pass time 8.43 +[May 1, 2026 - 04:19:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 305 : optimizer step time 0.40 +[May 1, 2026 - 04:19:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 305-------------------- +[May 1, 2026 - 04:19:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 170569.6245 iter count: 1.0 +[May 1, 2026 - 04:19:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6410 iter count: 1.0 +[May 1, 2026 - 04:19:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 170569.6245iter count: 1.0 +[May 1, 2026 - 04:19:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:19:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.5697iter count: 1.0 +[May 1, 2026 - 04:19:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.6758 iter count: 1.0 +[May 1, 2026 - 04:19:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.7303 iter count: 1.0 +[May 1, 2026 - 04:19:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 291027.0938iter count: 1.0 +[May 1, 2026 - 04:19:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 28.5458iter count: 1.0 +[May 1, 2026 - 04:19:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 5181888.0000iter count: 1.0 +[May 1, 2026 - 04:19:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6410iter count: 1.0 +[May 1, 2026 - 04:19:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.9683 iter count: 1.0 +[May 1, 2026 - 04:19:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 152756.6875iter count: 1.0 +[May 1, 2026 - 04:19:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.0082 iter count: 1.0 +[May 1, 2026 - 04:19:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 22392.2441iter count: 1.0 +[May 1, 2026 - 04:19:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 6.0801 iter count: 1.0 +[May 1, 2026 - 04:19:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 191036.3906iter count: 1.0 +[May 1, 2026 - 04:19:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 2.9658 iter count: 1.0 +[May 1, 2026 - 04:19:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 376.7635iter count: 1.0 +[May 1, 2026 - 04:19:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 04:19:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:19:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 04:20:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452563 41.610832 41.323044 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.818359 67.914062 67.531250 +util 96.250000 100.000000 92.000000 +[May 1, 2026 - 04:20:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 306 : avg iteration time 45.19 seconds +[May 1, 2026 - 04:20:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 306 : data loading time 0.00 +[May 1, 2026 - 04:20:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 306 : avg forward pass time 11.65 +[May 1, 2026 - 04:20:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 306 : backward pass time 10.15 +[May 1, 2026 - 04:20:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 306 : optimizer step time 1.58 +[May 1, 2026 - 04:20:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 306-------------------- +[May 1, 2026 - 04:20:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 04:20:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 04:20:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:20:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 04:21:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452683 41.611336 41.323044 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.662109 67.738281 67.433594 +util 95.250000 99.000000 92.000000 +[May 1, 2026 - 04:21:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 307 : avg iteration time 46.28 seconds +[May 1, 2026 - 04:21:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 307 : data loading time 0.00 +[May 1, 2026 - 04:21:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 307 : avg forward pass time 12.19 +[May 1, 2026 - 04:21:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 307 : backward pass time 10.16 +[May 1, 2026 - 04:21:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 307 : optimizer step time 1.57 +[May 1, 2026 - 04:21:21 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 307-------------------- +[May 1, 2026 - 04:21:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0104 iter count: 1.0 +[May 1, 2026 - 04:21:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 1, 2026 - 04:21:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:21:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 04:22:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452692 41.611336 41.323067 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.823242 67.914062 67.550781 +util 90.500000 95.000000 87.000000 +[May 1, 2026 - 04:22:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 308 : avg iteration time 46.33 seconds +[May 1, 2026 - 04:22:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 308 : data loading time 0.00 +[May 1, 2026 - 04:22:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 308 : avg forward pass time 12.23 +[May 1, 2026 - 04:22:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 308 : backward pass time 10.16 +[May 1, 2026 - 04:22:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 308 : optimizer step time 1.57 +[May 1, 2026 - 04:22:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 308-------------------- +[May 1, 2026 - 04:22:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0084 iter count: 1.0 +[May 1, 2026 - 04:22:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[May 1, 2026 - 04:22:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:22:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 04:22:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.453003 41.612579 41.323067 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.852539 67.992188 67.433594 +util 93.750000 98.000000 90.000000 +[May 1, 2026 - 04:22:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 309 : avg iteration time 46.24 seconds +[May 1, 2026 - 04:22:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 309 : data loading time 0.00 +[May 1, 2026 - 04:22:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 309 : avg forward pass time 12.18 +[May 1, 2026 - 04:22:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 309 : backward pass time 10.13 +[May 1, 2026 - 04:22:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 309 : optimizer step time 1.57 +[May 1, 2026 - 04:22:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 309-------------------- +[May 1, 2026 - 04:22:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 1, 2026 - 04:22:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 1, 2026 - 04:22:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:22:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 04:25:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452773 41.611683 41.323044 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.026367 136.076172 136.009766 +util 68.250000 72.000000 65.000000 +[May 1, 2026 - 04:25:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 310 : avg iteration time 163.48 seconds +[May 1, 2026 - 04:25:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 310 : data loading time 0.00 +[May 1, 2026 - 04:25:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 310 : avg forward pass time 73.13 +[May 1, 2026 - 04:25:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 310 : backward pass time 8.44 +[May 1, 2026 - 04:25:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 310 : optimizer step time 0.40 +[May 1, 2026 - 04:25:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 310-------------------- +[May 1, 2026 - 04:25:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 190663.9004 iter count: 1.0 +[May 1, 2026 - 04:25:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5856 iter count: 1.0 +[May 1, 2026 - 04:25:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 190663.9004iter count: 1.0 +[May 1, 2026 - 04:25:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:25:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.9083iter count: 1.0 +[May 1, 2026 - 04:25:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 2.7359 iter count: 1.0 +[May 1, 2026 - 04:25:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.6443 iter count: 1.0 +[May 1, 2026 - 04:25:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 317407.8125iter count: 1.0 +[May 1, 2026 - 04:25:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 237.8653iter count: 1.0 +[May 1, 2026 - 04:25:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 4362800.5000iter count: 1.0 +[May 1, 2026 - 04:25:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5856iter count: 1.0 +[May 1, 2026 - 04:25:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.7026 iter count: 1.0 +[May 1, 2026 - 04:25:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 12151.9023iter count: 1.0 +[May 1, 2026 - 04:25:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.3924 iter count: 1.0 +[May 1, 2026 - 04:25:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 48281.3906iter count: 1.0 +[May 1, 2026 - 04:25:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 6.9944 iter count: 1.0 +[May 1, 2026 - 04:25:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 1189302.2500iter count: 1.0 +[May 1, 2026 - 04:25:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.6067 iter count: 1.0 +[May 1, 2026 - 04:25:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 10030.0488iter count: 1.0 +[May 1, 2026 - 04:25:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 04:25:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:25:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 04:26:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452773 41.611683 41.323044 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.847168 67.992188 67.412109 +util 96.250000 99.000000 93.000000 +[May 1, 2026 - 04:26:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 311 : avg iteration time 48.16 seconds +[May 1, 2026 - 04:26:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 311 : data loading time 0.00 +[May 1, 2026 - 04:26:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 311 : avg forward pass time 13.14 +[May 1, 2026 - 04:26:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 311 : backward pass time 10.14 +[May 1, 2026 - 04:26:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 311 : optimizer step time 1.58 +[May 1, 2026 - 04:26:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 311-------------------- +[May 1, 2026 - 04:26:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 04:26:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 04:26:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:26:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 04:27:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452780 41.611683 41.323044 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.862305 67.992188 67.472656 +util 95.750000 100.000000 91.000000 +[May 1, 2026 - 04:27:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 312 : avg iteration time 46.30 seconds +[May 1, 2026 - 04:27:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 312 : data loading time 0.00 +[May 1, 2026 - 04:27:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 312 : avg forward pass time 12.23 +[May 1, 2026 - 04:27:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 312 : backward pass time 10.14 +[May 1, 2026 - 04:27:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 312 : optimizer step time 1.57 +[May 1, 2026 - 04:27:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 312-------------------- +[May 1, 2026 - 04:27:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 04:27:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 04:27:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:27:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 04:27:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452785 41.611683 41.323067 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.857422 67.992188 67.453125 +util 97.750000 100.000000 95.000000 +[May 1, 2026 - 04:27:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 313 : avg iteration time 46.32 seconds +[May 1, 2026 - 04:27:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 313 : data loading time 0.00 +[May 1, 2026 - 04:27:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 313 : avg forward pass time 12.22 +[May 1, 2026 - 04:27:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 313 : backward pass time 10.15 +[May 1, 2026 - 04:27:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 313 : optimizer step time 1.57 +[May 1, 2026 - 04:27:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 313-------------------- +[May 1, 2026 - 04:27:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0104 iter count: 1.0 +[May 1, 2026 - 04:27:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 1, 2026 - 04:27:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:27:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 04:28:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452904 41.612179 41.323044 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.857422 67.953125 67.570312 +util 98.000000 100.000000 96.000000 +[May 1, 2026 - 04:28:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 314 : avg iteration time 43.61 seconds +[May 1, 2026 - 04:28:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 314 : data loading time 0.00 +[May 1, 2026 - 04:28:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 314 : avg forward pass time 10.86 +[May 1, 2026 - 04:28:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 314 : backward pass time 10.14 +[May 1, 2026 - 04:28:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 314 : optimizer step time 1.58 +[May 1, 2026 - 04:28:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 314-------------------- +[May 1, 2026 - 04:28:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0104 iter count: 1.0 +[May 1, 2026 - 04:28:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 1, 2026 - 04:28:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:28:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 04:31:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452686 41.611309 41.323044 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.081543 136.109375 135.998047 +util 64.500000 69.000000 61.000000 +[May 1, 2026 - 04:31:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 315 : avg iteration time 166.15 seconds +[May 1, 2026 - 04:31:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 315 : data loading time 0.00 +[May 1, 2026 - 04:31:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 315 : avg forward pass time 74.45 +[May 1, 2026 - 04:31:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 315 : backward pass time 8.41 +[May 1, 2026 - 04:31:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 315 : optimizer step time 0.39 +[May 1, 2026 - 04:31:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 315-------------------- +[May 1, 2026 - 04:31:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 502842.6504 iter count: 1.0 +[May 1, 2026 - 04:31:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6132 iter count: 1.0 +[May 1, 2026 - 04:31:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 502842.6504iter count: 1.0 +[May 1, 2026 - 04:31:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:31:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.1574iter count: 1.0 +[May 1, 2026 - 04:31:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.8417 iter count: 1.0 +[May 1, 2026 - 04:31:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.3759 iter count: 1.0 +[May 1, 2026 - 04:31:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 967632.4375iter count: 1.0 +[May 1, 2026 - 04:31:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 5.3839 iter count: 1.0 +[May 1, 2026 - 04:31:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 18844216.0000iter count: 1.0 +[May 1, 2026 - 04:31:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6132iter count: 1.0 +[May 1, 2026 - 04:31:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 0.8417 iter count: 1.0 +[May 1, 2026 - 04:31:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 5.3839 iter count: 1.0 +[May 1, 2026 - 04:31:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.3835 iter count: 1.0 +[May 1, 2026 - 04:31:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 47426.9023iter count: 1.0 +[May 1, 2026 - 04:31:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.6592 iter count: 1.0 +[May 1, 2026 - 04:31:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 11140.0303iter count: 1.0 +[May 1, 2026 - 04:31:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.7571 iter count: 1.0 +[May 1, 2026 - 04:31:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 100131.2812iter count: 1.0 +[May 1, 2026 - 04:31:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 1, 2026 - 04:31:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:31:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 04:32:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452814 41.611816 41.323044 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.842285 67.933594 67.568359 +util 96.250000 100.000000 92.000000 +[May 1, 2026 - 04:32:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 316 : avg iteration time 47.94 seconds +[May 1, 2026 - 04:32:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 316 : data loading time 0.00 +[May 1, 2026 - 04:32:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 316 : avg forward pass time 13.03 +[May 1, 2026 - 04:32:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 316 : backward pass time 10.16 +[May 1, 2026 - 04:32:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 316 : optimizer step time 1.56 +[May 1, 2026 - 04:32:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 316-------------------- +[May 1, 2026 - 04:32:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0138 iter count: 1.0 +[May 1, 2026 - 04:32:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0138 iter count: 1.0 +[May 1, 2026 - 04:32:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:32:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 04:32:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452817 41.611797 41.323055 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.666504 67.718750 67.509766 +util 95.000000 99.000000 90.000000 +[May 1, 2026 - 04:32:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 317 : avg iteration time 43.58 seconds +[May 1, 2026 - 04:32:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 317 : data loading time 0.00 +[May 1, 2026 - 04:32:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 317 : avg forward pass time 10.85 +[May 1, 2026 - 04:32:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 317 : backward pass time 10.14 +[May 1, 2026 - 04:32:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 317 : optimizer step time 1.58 +[May 1, 2026 - 04:32:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 317-------------------- +[May 1, 2026 - 04:32:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 1, 2026 - 04:32:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 1, 2026 - 04:32:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:32:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 04:33:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452826 41.611797 41.323067 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.681152 67.718750 67.568359 +util 96.750000 100.000000 94.000000 +[May 1, 2026 - 04:33:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 318 : avg iteration time 46.31 seconds +[May 1, 2026 - 04:33:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 318 : data loading time 0.00 +[May 1, 2026 - 04:33:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 318 : avg forward pass time 12.22 +[May 1, 2026 - 04:33:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 318 : backward pass time 10.14 +[May 1, 2026 - 04:33:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 318 : optimizer step time 1.57 +[May 1, 2026 - 04:33:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 318-------------------- +[May 1, 2026 - 04:33:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 04:33:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 04:33:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:33:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 04:34:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452975 41.612293 41.323154 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.690918 67.738281 67.548828 +util 92.750000 97.000000 89.000000 +[May 1, 2026 - 04:34:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 319 : avg iteration time 43.74 seconds +[May 1, 2026 - 04:34:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 319 : data loading time 0.00 +[May 1, 2026 - 04:34:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 319 : avg forward pass time 10.94 +[May 1, 2026 - 04:34:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 319 : backward pass time 10.13 +[May 1, 2026 - 04:34:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 319 : optimizer step time 1.57 +[May 1, 2026 - 04:34:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 319-------------------- +[May 1, 2026 - 04:34:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 04:34:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 04:34:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:34:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 04:37:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452201 41.609219 41.323132 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.006836 136.029297 135.939453 +util 67.000000 71.000000 63.000000 +[May 1, 2026 - 04:37:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 320 : avg iteration time 168.67 seconds +[May 1, 2026 - 04:37:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 320 : data loading time 0.00 +[May 1, 2026 - 04:37:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 320 : avg forward pass time 75.73 +[May 1, 2026 - 04:37:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 320 : backward pass time 8.42 +[May 1, 2026 - 04:37:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 320 : optimizer step time 0.40 +[May 1, 2026 - 04:37:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 320-------------------- +[May 1, 2026 - 04:37:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 966508.3770 iter count: 1.0 +[May 1, 2026 - 04:37:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6747 iter count: 1.0 +[May 1, 2026 - 04:37:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 966508.3770iter count: 1.0 +[May 1, 2026 - 04:37:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:37:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 5.3501iter count: 1.0 +[May 1, 2026 - 04:37:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.8906 iter count: 1.0 +[May 1, 2026 - 04:37:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.3159 iter count: 1.0 +[May 1, 2026 - 04:37:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 1391638.0000iter count: 1.0 +[May 1, 2026 - 04:37:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 5.9369 iter count: 1.0 +[May 1, 2026 - 04:37:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 16715315.0000iter count: 1.0 +[May 1, 2026 - 04:37:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6747iter count: 1.0 +[May 1, 2026 - 04:37:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 1.1313 iter count: 1.0 +[May 1, 2026 - 04:37:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 9.6082 iter count: 1.0 +[May 1, 2026 - 04:37:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.3829 iter count: 1.0 +[May 1, 2026 - 04:37:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 47374.4648iter count: 1.0 +[May 1, 2026 - 04:37:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.8488 iter count: 1.0 +[May 1, 2026 - 04:37:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 16278.8545iter count: 1.0 +[May 1, 2026 - 04:37:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.2912 iter count: 1.0 +[May 1, 2026 - 04:37:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 291384.6250iter count: 1.0 +[May 1, 2026 - 04:37:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 1, 2026 - 04:37:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:37:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 04:38:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452332 41.609722 41.323132 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.793457 67.835938 67.666016 +util 95.000000 100.000000 89.000000 +[May 1, 2026 - 04:38:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 321 : avg iteration time 45.47 seconds +[May 1, 2026 - 04:38:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 321 : data loading time 0.00 +[May 1, 2026 - 04:38:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 321 : avg forward pass time 11.82 +[May 1, 2026 - 04:38:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 321 : backward pass time 10.14 +[May 1, 2026 - 04:38:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 321 : optimizer step time 1.56 +[May 1, 2026 - 04:38:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 321-------------------- +[May 1, 2026 - 04:38:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0108 iter count: 1.0 +[May 1, 2026 - 04:38:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 1, 2026 - 04:38:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:38:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 04:38:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452579 41.610729 41.323132 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.812988 67.914062 67.509766 +util 98.000000 100.000000 96.000000 +[May 1, 2026 - 04:38:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 322 : avg iteration time 48.94 seconds +[May 1, 2026 - 04:38:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 322 : data loading time 0.00 +[May 1, 2026 - 04:38:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 322 : avg forward pass time 13.54 +[May 1, 2026 - 04:38:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 322 : backward pass time 10.14 +[May 1, 2026 - 04:38:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 322 : optimizer step time 1.58 +[May 1, 2026 - 04:38:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 322-------------------- +[May 1, 2026 - 04:38:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 04:38:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 04:38:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:38:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 04:39:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452660 41.610981 41.323158 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.734863 67.816406 67.490234 +util 95.000000 99.000000 92.000000 +[May 1, 2026 - 04:39:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 323 : avg iteration time 48.95 seconds +[May 1, 2026 - 04:39:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 323 : data loading time 0.00 +[May 1, 2026 - 04:39:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 323 : avg forward pass time 13.55 +[May 1, 2026 - 04:39:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 323 : backward pass time 10.15 +[May 1, 2026 - 04:39:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 323 : optimizer step time 1.57 +[May 1, 2026 - 04:39:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 323-------------------- +[May 1, 2026 - 04:39:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 1, 2026 - 04:39:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 1, 2026 - 04:39:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:39:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 04:40:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452622 41.610897 41.323132 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.842285 67.933594 67.568359 +util 95.250000 100.000000 90.000000 +[May 1, 2026 - 04:40:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 324 : avg iteration time 43.73 seconds +[May 1, 2026 - 04:40:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 324 : data loading time 0.00 +[May 1, 2026 - 04:40:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 324 : avg forward pass time 10.92 +[May 1, 2026 - 04:40:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 324 : backward pass time 10.15 +[May 1, 2026 - 04:40:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 324 : optimizer step time 1.57 +[May 1, 2026 - 04:40:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 324-------------------- +[May 1, 2026 - 04:40:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[May 1, 2026 - 04:40:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 1, 2026 - 04:40:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:40:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 04:43:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452497 41.610374 41.323132 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.037598 136.050781 135.998047 +util 60.000000 65.000000 57.000000 +[May 1, 2026 - 04:43:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 325 : avg iteration time 163.08 seconds +[May 1, 2026 - 04:43:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 325 : data loading time 0.00 +[May 1, 2026 - 04:43:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 325 : avg forward pass time 72.93 +[May 1, 2026 - 04:43:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 325 : backward pass time 8.42 +[May 1, 2026 - 04:43:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 325 : optimizer step time 0.42 +[May 1, 2026 - 04:43:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 325-------------------- +[May 1, 2026 - 04:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 260458.8555 iter count: 1.0 +[May 1, 2026 - 04:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5665 iter count: 1.0 +[May 1, 2026 - 04:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 260458.8555iter count: 1.0 +[May 1, 2026 - 04:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.8711iter count: 1.0 +[May 1, 2026 - 04:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.6101 iter count: 1.0 +[May 1, 2026 - 04:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.5905 iter count: 1.0 +[May 1, 2026 - 04:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 479187.9062iter count: 1.0 +[May 1, 2026 - 04:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 3.3875 iter count: 1.0 +[May 1, 2026 - 04:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 3917407.2500iter count: 1.0 +[May 1, 2026 - 04:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5665iter count: 1.0 +[May 1, 2026 - 04:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.3391 iter count: 1.0 +[May 1, 2026 - 04:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 794.8444iter count: 1.0 +[May 1, 2026 - 04:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.6351 iter count: 1.0 +[May 1, 2026 - 04:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 10616.6191iter count: 1.0 +[May 1, 2026 - 04:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.8346 iter count: 1.0 +[May 1, 2026 - 04:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 116926.3125iter count: 1.0 +[May 1, 2026 - 04:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.0216 iter count: 1.0 +[May 1, 2026 - 04:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 23000.9141iter count: 1.0 +[May 1, 2026 - 04:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 1, 2026 - 04:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:43:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 04:43:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452499 41.610374 41.323135 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.856934 67.953125 67.568359 +util 94.000000 98.000000 91.000000 +[May 1, 2026 - 04:43:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 326 : avg iteration time 47.81 seconds +[May 1, 2026 - 04:43:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 326 : data loading time 0.00 +[May 1, 2026 - 04:43:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 326 : avg forward pass time 12.98 +[May 1, 2026 - 04:43:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 326 : backward pass time 10.14 +[May 1, 2026 - 04:43:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 326 : optimizer step time 1.57 +[May 1, 2026 - 04:43:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 326-------------------- +[May 1, 2026 - 04:43:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 04:43:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 04:43:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:43:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 04:44:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452622 41.610870 41.323135 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.842285 67.933594 67.568359 +util 95.750000 99.000000 93.000000 +[May 1, 2026 - 04:44:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 327 : avg iteration time 49.02 seconds +[May 1, 2026 - 04:44:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 327 : data loading time 0.00 +[May 1, 2026 - 04:44:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 327 : avg forward pass time 13.58 +[May 1, 2026 - 04:44:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 327 : backward pass time 10.15 +[May 1, 2026 - 04:44:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 327 : optimizer step time 1.57 +[May 1, 2026 - 04:44:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 327-------------------- +[May 1, 2026 - 04:44:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 04:44:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 04:44:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:44:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 04:45:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452627 41.610870 41.323135 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.856934 67.953125 67.568359 +util 93.250000 99.000000 87.000000 +[May 1, 2026 - 04:45:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 328 : avg iteration time 49.07 seconds +[May 1, 2026 - 04:45:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 328 : data loading time 0.00 +[May 1, 2026 - 04:45:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 328 : avg forward pass time 13.59 +[May 1, 2026 - 04:45:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 328 : backward pass time 10.16 +[May 1, 2026 - 04:45:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 328 : optimizer step time 1.56 +[May 1, 2026 - 04:45:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 328-------------------- +[May 1, 2026 - 04:45:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 1, 2026 - 04:45:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 1, 2026 - 04:45:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:45:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 04:46:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452527 41.610489 41.323135 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.886230 67.992188 67.568359 +util 97.500000 100.000000 95.000000 +[May 1, 2026 - 04:46:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 329 : avg iteration time 46.36 seconds +[May 1, 2026 - 04:46:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 329 : data loading time 0.00 +[May 1, 2026 - 04:46:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 329 : avg forward pass time 12.25 +[May 1, 2026 - 04:46:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 329 : backward pass time 10.15 +[May 1, 2026 - 04:46:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 329 : optimizer step time 1.57 +[May 1, 2026 - 04:46:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 329-------------------- +[May 1, 2026 - 04:46:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 04:46:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 04:46:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:46:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 04:49:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452426 41.610062 41.323135 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.076660 136.095703 136.070312 +util 63.750000 68.000000 61.000000 +[May 1, 2026 - 04:49:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 330 : avg iteration time 171.37 seconds +[May 1, 2026 - 04:49:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 330 : data loading time 0.00 +[May 1, 2026 - 04:49:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 330 : avg forward pass time 77.07 +[May 1, 2026 - 04:49:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 330 : backward pass time 8.43 +[May 1, 2026 - 04:49:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 330 : optimizer step time 0.40 +[May 1, 2026 - 04:49:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 330-------------------- +[May 1, 2026 - 04:49:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 63685.0083 iter count: 1.0 +[May 1, 2026 - 04:49:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6240 iter count: 1.0 +[May 1, 2026 - 04:49:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 63685.0083iter count: 1.0 +[May 1, 2026 - 04:49:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:49:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.3031iter count: 1.0 +[May 1, 2026 - 04:49:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.1630 iter count: 1.0 +[May 1, 2026 - 04:49:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 6.9528 iter count: 1.0 +[May 1, 2026 - 04:49:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 85739.2734iter count: 1.0 +[May 1, 2026 - 04:49:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 10.2362iter count: 1.0 +[May 1, 2026 - 04:49:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 1094281.1250iter count: 1.0 +[May 1, 2026 - 04:49:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6240iter count: 1.0 +[May 1, 2026 - 04:49:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.8717 iter count: 1.0 +[May 1, 2026 - 04:49:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 2306.0862iter count: 1.0 +[May 1, 2026 - 04:49:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 2.5912 iter count: 1.0 +[May 1, 2026 - 04:49:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 178.1162iter count: 1.0 +[May 1, 2026 - 04:49:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.1727 iter count: 1.0 +[May 1, 2026 - 04:49:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 569.9119iter count: 1.0 +[May 1, 2026 - 04:49:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.5415 iter count: 1.0 +[May 1, 2026 - 04:49:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 1191.5084iter count: 1.0 +[May 1, 2026 - 04:49:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 04:49:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:49:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 04:49:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452426 41.610062 41.323135 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.072266 68.207031 67.667969 +util 97.750000 100.000000 96.000000 +[May 1, 2026 - 04:49:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 331 : avg iteration time 47.85 seconds +[May 1, 2026 - 04:49:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 331 : data loading time 0.00 +[May 1, 2026 - 04:49:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 331 : avg forward pass time 12.99 +[May 1, 2026 - 04:49:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 331 : backward pass time 10.15 +[May 1, 2026 - 04:49:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 331 : optimizer step time 1.57 +[May 1, 2026 - 04:49:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 331-------------------- +[May 1, 2026 - 04:49:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0101 iter count: 1.0 +[May 1, 2026 - 04:49:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 1, 2026 - 04:49:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:49:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 04:50:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452617 41.610847 41.323135 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.964844 68.070312 67.648438 +util 96.250000 99.000000 89.000000 +[May 1, 2026 - 04:50:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 332 : avg iteration time 49.02 seconds +[May 1, 2026 - 04:50:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 332 : data loading time 0.00 +[May 1, 2026 - 04:50:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 332 : avg forward pass time 13.57 +[May 1, 2026 - 04:50:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 332 : backward pass time 10.14 +[May 1, 2026 - 04:50:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 332 : optimizer step time 1.57 +[May 1, 2026 - 04:50:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 332-------------------- +[May 1, 2026 - 04:50:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 04:50:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 04:50:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:50:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 04:51:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452606 41.610802 41.323135 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.969727 68.070312 67.667969 +util 94.000000 99.000000 88.000000 +[May 1, 2026 - 04:51:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 333 : avg iteration time 43.62 seconds +[May 1, 2026 - 04:51:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 333 : data loading time 0.00 +[May 1, 2026 - 04:51:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 333 : avg forward pass time 10.88 +[May 1, 2026 - 04:51:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 333 : backward pass time 10.15 +[May 1, 2026 - 04:51:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 333 : optimizer step time 1.56 +[May 1, 2026 - 04:51:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 333-------------------- +[May 1, 2026 - 04:51:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 1, 2026 - 04:51:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 1, 2026 - 04:51:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:51:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 04:52:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452605 41.610802 41.323135 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.969727 68.070312 67.667969 +util 95.750000 99.000000 93.000000 +[May 1, 2026 - 04:52:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 334 : avg iteration time 48.92 seconds +[May 1, 2026 - 04:52:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 334 : data loading time 0.00 +[May 1, 2026 - 04:52:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 334 : avg forward pass time 13.52 +[May 1, 2026 - 04:52:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 334 : backward pass time 10.16 +[May 1, 2026 - 04:52:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 334 : optimizer step time 1.56 +[May 1, 2026 - 04:52:21 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 334-------------------- +[May 1, 2026 - 04:52:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0101 iter count: 1.0 +[May 1, 2026 - 04:52:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 1, 2026 - 04:52:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:52:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 04:55:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452445 41.610111 41.323135 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.066895 136.089844 135.998047 +util 62.000000 72.000000 54.000000 +[May 1, 2026 - 04:55:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 335 : avg iteration time 168.75 seconds +[May 1, 2026 - 04:55:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 335 : data loading time 0.00 +[May 1, 2026 - 04:55:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 335 : avg forward pass time 75.78 +[May 1, 2026 - 04:55:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 335 : backward pass time 8.40 +[May 1, 2026 - 04:55:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 335 : optimizer step time 0.41 +[May 1, 2026 - 04:55:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 335-------------------- +[May 1, 2026 - 04:55:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 307821.3047 iter count: 1.0 +[May 1, 2026 - 04:55:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6320 iter count: 1.0 +[May 1, 2026 - 04:55:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 307821.3047iter count: 1.0 +[May 1, 2026 - 04:55:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:55:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.5486iter count: 1.0 +[May 1, 2026 - 04:55:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.6142 iter count: 1.0 +[May 1, 2026 - 04:55:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.9066 iter count: 1.0 +[May 1, 2026 - 04:55:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 455552.7188iter count: 1.0 +[May 1, 2026 - 04:55:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 3.4157 iter count: 1.0 +[May 1, 2026 - 04:55:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 7371398.5000iter count: 1.0 +[May 1, 2026 - 04:55:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6320iter count: 1.0 +[May 1, 2026 - 04:55:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.1949 iter count: 1.0 +[May 1, 2026 - 04:55:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 32526.9375iter count: 1.0 +[May 1, 2026 - 04:55:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.9985 iter count: 1.0 +[May 1, 2026 - 04:55:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 2972.0618iter count: 1.0 +[May 1, 2026 - 04:55:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.4284 iter count: 1.0 +[May 1, 2026 - 04:55:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 7021.8833iter count: 1.0 +[May 1, 2026 - 04:55:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.4993 iter count: 1.0 +[May 1, 2026 - 04:55:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 8092.0254iter count: 1.0 +[May 1, 2026 - 04:55:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 04:55:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:55:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 04:55:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452472 41.610123 41.323254 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.725586 67.777344 67.570312 +util 94.000000 98.000000 91.000000 +[May 1, 2026 - 04:55:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 336 : avg iteration time 47.84 seconds +[May 1, 2026 - 04:55:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 336 : data loading time 0.00 +[May 1, 2026 - 04:55:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 336 : avg forward pass time 12.98 +[May 1, 2026 - 04:55:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 336 : backward pass time 10.15 +[May 1, 2026 - 04:55:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 336 : optimizer step time 1.57 +[May 1, 2026 - 04:55:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 336-------------------- +[May 1, 2026 - 04:55:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 1, 2026 - 04:55:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 1, 2026 - 04:55:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:55:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 04:56:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452647 41.610847 41.323254 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.837891 67.953125 67.492188 +util 92.500000 96.000000 89.000000 +[May 1, 2026 - 04:56:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 337 : avg iteration time 43.58 seconds +[May 1, 2026 - 04:56:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 337 : data loading time 0.00 +[May 1, 2026 - 04:56:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 337 : avg forward pass time 10.85 +[May 1, 2026 - 04:56:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 337 : backward pass time 10.15 +[May 1, 2026 - 04:56:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 337 : optimizer step time 1.57 +[May 1, 2026 - 04:56:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 337-------------------- +[May 1, 2026 - 04:56:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 04:56:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 04:56:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:56:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 04:57:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452712 41.611099 41.323254 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.842773 67.933594 67.570312 +util 92.750000 97.000000 89.000000 +[May 1, 2026 - 04:57:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 338 : avg iteration time 49.00 seconds +[May 1, 2026 - 04:57:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 338 : data loading time 0.00 +[May 1, 2026 - 04:57:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 338 : avg forward pass time 13.57 +[May 1, 2026 - 04:57:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 338 : backward pass time 10.14 +[May 1, 2026 - 04:57:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 338 : optimizer step time 1.57 +[May 1, 2026 - 04:57:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 338-------------------- +[May 1, 2026 - 04:57:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0101 iter count: 1.0 +[May 1, 2026 - 04:57:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 1, 2026 - 04:57:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:57:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 04:58:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452566 41.610523 41.323254 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.813477 67.933594 67.453125 +util 98.000000 100.000000 96.000000 +[May 1, 2026 - 04:58:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 339 : avg iteration time 46.28 seconds +[May 1, 2026 - 04:58:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 339 : data loading time 0.00 +[May 1, 2026 - 04:58:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 339 : avg forward pass time 12.21 +[May 1, 2026 - 04:58:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 339 : backward pass time 10.15 +[May 1, 2026 - 04:58:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 339 : optimizer step time 1.55 +[May 1, 2026 - 04:58:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 339-------------------- +[May 1, 2026 - 04:58:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 04:58:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 04:58:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 04:58:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 05:00:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452437 41.609917 41.323322 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.062012 136.070312 136.037109 +util 64.500000 69.000000 61.000000 +[May 1, 2026 - 05:00:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 340 : avg iteration time 163.36 seconds +[May 1, 2026 - 05:00:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 340 : data loading time 0.00 +[May 1, 2026 - 05:00:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 340 : avg forward pass time 73.08 +[May 1, 2026 - 05:00:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 340 : backward pass time 8.42 +[May 1, 2026 - 05:00:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 340 : optimizer step time 0.39 +[May 1, 2026 - 05:00:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 340-------------------- +[May 1, 2026 - 05:00:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 97358.1260 iter count: 1.0 +[May 1, 2026 - 05:00:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6166 iter count: 1.0 +[May 1, 2026 - 05:00:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 97358.1260iter count: 1.0 +[May 1, 2026 - 05:00:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:00:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.9178iter count: 1.0 +[May 1, 2026 - 05:00:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 2.8681 iter count: 1.0 +[May 1, 2026 - 05:00:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.0192 iter count: 1.0 +[May 1, 2026 - 05:00:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 151285.7188iter count: 1.0 +[May 1, 2026 - 05:00:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 309.8948iter count: 1.0 +[May 1, 2026 - 05:00:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 1249611.7500iter count: 1.0 +[May 1, 2026 - 05:00:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6166iter count: 1.0 +[May 1, 2026 - 05:00:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.9912 iter count: 1.0 +[May 1, 2026 - 05:00:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 159900.1094iter count: 1.0 +[May 1, 2026 - 05:00:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 6.1564 iter count: 1.0 +[May 1, 2026 - 05:00:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 222532.2031iter count: 1.0 +[May 1, 2026 - 05:00:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.1039 iter count: 1.0 +[May 1, 2026 - 05:00:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 496.6491iter count: 1.0 +[May 1, 2026 - 05:00:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.7829 iter count: 1.0 +[May 1, 2026 - 05:00:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 1930.9646iter count: 1.0 +[May 1, 2026 - 05:00:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 1, 2026 - 05:00:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:00:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 05:01:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452436 41.609917 41.323338 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.735352 67.777344 67.609375 +util 94.500000 98.000000 91.000000 +[May 1, 2026 - 05:01:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 341 : avg iteration time 45.19 seconds +[May 1, 2026 - 05:01:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 341 : data loading time 0.00 +[May 1, 2026 - 05:01:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 341 : avg forward pass time 11.67 +[May 1, 2026 - 05:01:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 341 : backward pass time 10.14 +[May 1, 2026 - 05:01:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 341 : optimizer step time 1.56 +[May 1, 2026 - 05:01:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 341-------------------- +[May 1, 2026 - 05:01:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 05:01:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 05:01:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:01:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 05:02:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452808 41.611420 41.323322 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.803711 67.914062 67.472656 +util 93.250000 97.000000 90.000000 +[May 1, 2026 - 05:02:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 342 : avg iteration time 43.58 seconds +[May 1, 2026 - 05:02:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 342 : data loading time 0.00 +[May 1, 2026 - 05:02:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 342 : avg forward pass time 10.85 +[May 1, 2026 - 05:02:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 342 : backward pass time 10.16 +[May 1, 2026 - 05:02:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 342 : optimizer step time 1.58 +[May 1, 2026 - 05:02:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 342-------------------- +[May 1, 2026 - 05:02:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 05:02:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 05:02:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:02:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 05:03:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452829 41.611420 41.323322 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.813477 67.875000 67.628906 +util 96.500000 100.000000 93.000000 +[May 1, 2026 - 05:03:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 343 : avg iteration time 46.24 seconds +[May 1, 2026 - 05:03:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 343 : data loading time 0.00 +[May 1, 2026 - 05:03:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 343 : avg forward pass time 12.19 +[May 1, 2026 - 05:03:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 343 : backward pass time 10.14 +[May 1, 2026 - 05:03:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 343 : optimizer step time 1.57 +[May 1, 2026 - 05:03:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 343-------------------- +[May 1, 2026 - 05:03:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 05:03:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 05:03:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:03:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 05:04:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452831 41.611427 41.323322 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.803711 67.875000 67.589844 +util 97.250000 100.000000 94.000000 +[May 1, 2026 - 05:04:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 344 : avg iteration time 49.01 seconds +[May 1, 2026 - 05:04:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 344 : data loading time 0.00 +[May 1, 2026 - 05:04:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 344 : avg forward pass time 13.58 +[May 1, 2026 - 05:04:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 344 : backward pass time 10.15 +[May 1, 2026 - 05:04:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 344 : optimizer step time 1.56 +[May 1, 2026 - 05:04:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 344-------------------- +[May 1, 2026 - 05:04:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0075 iter count: 1.0 +[May 1, 2026 - 05:04:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0075 iter count: 1.0 +[May 1, 2026 - 05:04:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:04:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 05:06:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452431 41.609825 41.323322 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.006836 136.056641 135.990234 +util 64.250000 70.000000 58.000000 +[May 1, 2026 - 05:06:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 345 : avg iteration time 163.68 seconds +[May 1, 2026 - 05:06:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 345 : data loading time 0.00 +[May 1, 2026 - 05:06:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 345 : avg forward pass time 73.23 +[May 1, 2026 - 05:06:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 345 : backward pass time 8.42 +[May 1, 2026 - 05:06:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 345 : optimizer step time 0.39 +[May 1, 2026 - 05:06:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 345-------------------- +[May 1, 2026 - 05:06:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 866838.3066 iter count: 1.0 +[May 1, 2026 - 05:06:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5699 iter count: 1.0 +[May 1, 2026 - 05:06:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 866838.3066iter count: 1.0 +[May 1, 2026 - 05:06:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:06:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.6893iter count: 1.0 +[May 1, 2026 - 05:06:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.6276 iter count: 1.0 +[May 1, 2026 - 05:06:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.3870 iter count: 1.0 +[May 1, 2026 - 05:06:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 1495380.5000iter count: 1.0 +[May 1, 2026 - 05:06:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 3.5083 iter count: 1.0 +[May 1, 2026 - 05:06:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 19270574.0000iter count: 1.0 +[May 1, 2026 - 05:06:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5699iter count: 1.0 +[May 1, 2026 - 05:06:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 1.6495 iter count: 1.0 +[May 1, 2026 - 05:06:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 27.0859 iter count: 1.0 +[May 1, 2026 - 05:06:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.5495 iter count: 1.0 +[May 1, 2026 - 05:06:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 66102.6328iter count: 1.0 +[May 1, 2026 - 05:06:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.0146 iter count: 1.0 +[May 1, 2026 - 05:06:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 3069.0735iter count: 1.0 +[May 1, 2026 - 05:06:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 1.0610 iter count: 1.0 +[May 1, 2026 - 05:06:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 8.3484 iter count: 1.0 +[May 1, 2026 - 05:06:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[May 1, 2026 - 05:06:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:06:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 05:07:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452442 41.609871 41.323322 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.925781 67.953125 67.843750 +util 95.500000 99.000000 91.000000 +[May 1, 2026 - 05:07:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 346 : avg iteration time 50.62 seconds +[May 1, 2026 - 05:07:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 346 : data loading time 0.00 +[May 1, 2026 - 05:07:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 346 : avg forward pass time 14.38 +[May 1, 2026 - 05:07:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 346 : backward pass time 10.13 +[May 1, 2026 - 05:07:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 346 : optimizer step time 1.57 +[May 1, 2026 - 05:07:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 346-------------------- +[May 1, 2026 - 05:07:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0108 iter count: 1.0 +[May 1, 2026 - 05:07:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 1, 2026 - 05:07:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:07:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 05:08:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452678 41.610813 41.323322 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.940430 68.019531 67.914062 +util 97.000000 100.000000 94.000000 +[May 1, 2026 - 05:08:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 347 : avg iteration time 46.31 seconds +[May 1, 2026 - 05:08:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 347 : data loading time 0.00 +[May 1, 2026 - 05:08:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 347 : avg forward pass time 12.22 +[May 1, 2026 - 05:08:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 347 : backward pass time 10.15 +[May 1, 2026 - 05:08:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 347 : optimizer step time 1.57 +[May 1, 2026 - 05:08:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 347-------------------- +[May 1, 2026 - 05:08:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0108 iter count: 1.0 +[May 1, 2026 - 05:08:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 1, 2026 - 05:08:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:08:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 05:09:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452741 41.611065 41.323322 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.945312 68.039062 67.914062 +util 98.500000 100.000000 97.000000 +[May 1, 2026 - 05:09:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 348 : avg iteration time 49.02 seconds +[May 1, 2026 - 05:09:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 348 : data loading time 0.00 +[May 1, 2026 - 05:09:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 348 : avg forward pass time 13.59 +[May 1, 2026 - 05:09:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 348 : backward pass time 10.15 +[May 1, 2026 - 05:09:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 348 : optimizer step time 1.57 +[May 1, 2026 - 05:09:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 348-------------------- +[May 1, 2026 - 05:09:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 05:09:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 05:09:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:09:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 05:09:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452744 41.611065 41.323322 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.096680 68.109375 68.058594 +util 96.000000 100.000000 88.000000 +[May 1, 2026 - 05:09:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 349 : avg iteration time 43.61 seconds +[May 1, 2026 - 05:09:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 349 : data loading time 0.00 +[May 1, 2026 - 05:09:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 349 : avg forward pass time 10.87 +[May 1, 2026 - 05:09:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 349 : backward pass time 10.14 +[May 1, 2026 - 05:09:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 349 : optimizer step time 1.58 +[May 1, 2026 - 05:09:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 349-------------------- +[May 1, 2026 - 05:09:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 05:09:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 05:09:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:09:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 05:12:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452198 41.608868 41.323322 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.050781 136.068359 135.998047 +util 61.500000 66.000000 58.000000 +[May 1, 2026 - 05:12:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 350 : avg iteration time 163.51 seconds +[May 1, 2026 - 05:12:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 350 : data loading time 0.00 +[May 1, 2026 - 05:12:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 350 : avg forward pass time 73.13 +[May 1, 2026 - 05:12:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 350 : backward pass time 8.42 +[May 1, 2026 - 05:12:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 350 : optimizer step time 0.40 +[May 1, 2026 - 05:12:40 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 350-------------------- +[May 1, 2026 - 05:12:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 379375.8711 iter count: 1.0 +[May 1, 2026 - 05:12:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6011 iter count: 1.0 +[May 1, 2026 - 05:12:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 379375.8711iter count: 1.0 +[May 1, 2026 - 05:12:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:12:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.2151iter count: 1.0 +[May 1, 2026 - 05:12:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.9136 iter count: 1.0 +[May 1, 2026 - 05:12:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.3259 iter count: 1.0 +[May 1, 2026 - 05:12:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 622794.6875iter count: 1.0 +[May 1, 2026 - 05:12:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 6.2160 iter count: 1.0 +[May 1, 2026 - 05:12:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 17053190.0000iter count: 1.0 +[May 1, 2026 - 05:12:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6011iter count: 1.0 +[May 1, 2026 - 05:12:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.3512 iter count: 1.0 +[May 1, 2026 - 05:12:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 814.4233iter count: 1.0 +[May 1, 2026 - 05:12:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.2786 iter count: 1.0 +[May 1, 2026 - 05:12:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 38453.8008iter count: 1.0 +[May 1, 2026 - 05:12:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 1.6675 iter count: 1.0 +[May 1, 2026 - 05:12:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 28.0768 iter count: 1.0 +[May 1, 2026 - 05:12:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.8368 iter count: 1.0 +[May 1, 2026 - 05:12:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 15892.5977iter count: 1.0 +[May 1, 2026 - 05:12:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 05:12:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:12:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 05:13:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452195 41.608845 41.323322 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.798828 68.039062 67.718750 +util 97.250000 100.000000 94.000000 +[May 1, 2026 - 05:13:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 351 : avg iteration time 47.86 seconds +[May 1, 2026 - 05:13:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 351 : data loading time 0.00 +[May 1, 2026 - 05:13:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 351 : avg forward pass time 12.99 +[May 1, 2026 - 05:13:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 351 : backward pass time 10.14 +[May 1, 2026 - 05:13:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 351 : optimizer step time 1.57 +[May 1, 2026 - 05:13:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 351-------------------- +[May 1, 2026 - 05:13:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 05:13:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 05:13:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:13:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 05:14:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452634 41.610600 41.323322 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.823242 67.953125 67.433594 +util 94.000000 100.000000 88.000000 +[May 1, 2026 - 05:14:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 352 : avg iteration time 46.25 seconds +[May 1, 2026 - 05:14:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 352 : data loading time 0.00 +[May 1, 2026 - 05:14:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 352 : avg forward pass time 12.19 +[May 1, 2026 - 05:14:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 352 : backward pass time 10.15 +[May 1, 2026 - 05:14:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 352 : optimizer step time 1.58 +[May 1, 2026 - 05:14:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 352-------------------- +[May 1, 2026 - 05:14:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 1, 2026 - 05:14:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 1, 2026 - 05:14:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:14:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 05:15:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452762 41.611092 41.323322 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.823242 67.953125 67.433594 +util 95.500000 99.000000 88.000000 +[May 1, 2026 - 05:15:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 353 : avg iteration time 48.98 seconds +[May 1, 2026 - 05:15:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 353 : data loading time 0.00 +[May 1, 2026 - 05:15:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 353 : avg forward pass time 13.56 +[May 1, 2026 - 05:15:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 353 : backward pass time 10.16 +[May 1, 2026 - 05:15:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 353 : optimizer step time 1.57 +[May 1, 2026 - 05:15:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 353-------------------- +[May 1, 2026 - 05:15:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0086 iter count: 1.0 +[May 1, 2026 - 05:15:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[May 1, 2026 - 05:15:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:15:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 05:15:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452758 41.611092 41.323322 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.842773 67.953125 67.511719 +util 95.250000 100.000000 90.000000 +[May 1, 2026 - 05:15:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 354 : avg iteration time 43.65 seconds +[May 1, 2026 - 05:15:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 354 : data loading time 0.00 +[May 1, 2026 - 05:15:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 354 : avg forward pass time 10.88 +[May 1, 2026 - 05:15:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 354 : backward pass time 10.15 +[May 1, 2026 - 05:15:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 354 : optimizer step time 1.58 +[May 1, 2026 - 05:15:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 354-------------------- +[May 1, 2026 - 05:15:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 05:15:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 05:15:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:15:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 05:18:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452133 41.608585 41.323322 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.080078 136.087891 136.056641 +util 61.000000 65.000000 57.000000 +[May 1, 2026 - 05:18:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 355 : avg iteration time 166.26 seconds +[May 1, 2026 - 05:18:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 355 : data loading time 0.00 +[May 1, 2026 - 05:18:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 355 : avg forward pass time 74.50 +[May 1, 2026 - 05:18:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 355 : backward pass time 8.41 +[May 1, 2026 - 05:18:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 355 : optimizer step time 0.40 +[May 1, 2026 - 05:18:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 355-------------------- +[May 1, 2026 - 05:18:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 214621.8389 iter count: 1.0 +[May 1, 2026 - 05:18:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5820 iter count: 1.0 +[May 1, 2026 - 05:18:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 214621.8389iter count: 1.0 +[May 1, 2026 - 05:18:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:18:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.8818iter count: 1.0 +[May 1, 2026 - 05:18:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.4888 iter count: 1.0 +[May 1, 2026 - 05:18:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.9325 iter count: 1.0 +[May 1, 2026 - 05:18:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 411648.9375iter count: 1.0 +[May 1, 2026 - 05:18:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 19.6425iter count: 1.0 +[May 1, 2026 - 05:18:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 7764571.5000iter count: 1.0 +[May 1, 2026 - 05:18:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5820iter count: 1.0 +[May 1, 2026 - 05:18:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 6.2246 iter count: 1.0 +[May 1, 2026 - 05:18:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 255038.5781iter count: 1.0 +[May 1, 2026 - 05:18:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.1699 iter count: 1.0 +[May 1, 2026 - 05:18:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 4187.1240iter count: 1.0 +[May 1, 2026 - 05:18:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.2127 iter count: 1.0 +[May 1, 2026 - 05:18:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 4561.1201iter count: 1.0 +[May 1, 2026 - 05:18:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.9587 iter count: 1.0 +[May 1, 2026 - 05:18:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 149851.2500iter count: 1.0 +[May 1, 2026 - 05:18:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 05:18:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:18:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 05:19:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452195 41.608829 41.323322 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.744629 67.796875 67.587891 +util 97.750000 100.000000 95.000000 +[May 1, 2026 - 05:19:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 356 : avg iteration time 47.86 seconds +[May 1, 2026 - 05:19:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 356 : data loading time 0.00 +[May 1, 2026 - 05:19:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 356 : avg forward pass time 13.00 +[May 1, 2026 - 05:19:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 356 : backward pass time 10.15 +[May 1, 2026 - 05:19:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 356 : optimizer step time 1.56 +[May 1, 2026 - 05:19:21 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 356-------------------- +[May 1, 2026 - 05:19:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 05:19:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 05:19:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:19:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 05:20:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452444 41.609833 41.323322 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.895996 67.992188 67.607422 +util 98.500000 100.000000 97.000000 +[May 1, 2026 - 05:20:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 357 : avg iteration time 43.62 seconds +[May 1, 2026 - 05:20:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 357 : data loading time 0.00 +[May 1, 2026 - 05:20:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 357 : avg forward pass time 10.87 +[May 1, 2026 - 05:20:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 357 : backward pass time 10.15 +[May 1, 2026 - 05:20:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 357 : optimizer step time 1.57 +[May 1, 2026 - 05:20:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 357-------------------- +[May 1, 2026 - 05:20:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 05:20:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 05:20:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:20:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 05:20:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452819 41.611332 41.323322 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.720215 67.796875 67.490234 +util 95.750000 100.000000 88.000000 +[May 1, 2026 - 05:20:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 358 : avg iteration time 43.61 seconds +[May 1, 2026 - 05:20:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 358 : data loading time 0.00 +[May 1, 2026 - 05:20:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 358 : avg forward pass time 10.86 +[May 1, 2026 - 05:20:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 358 : backward pass time 10.14 +[May 1, 2026 - 05:20:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 358 : optimizer step time 1.58 +[May 1, 2026 - 05:20:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 358-------------------- +[May 1, 2026 - 05:20:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 05:20:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 05:20:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:20:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 05:21:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452945 41.611835 41.323322 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.861816 68.011719 67.412109 +util 95.500000 99.000000 91.000000 +[May 1, 2026 - 05:21:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 359 : avg iteration time 43.66 seconds +[May 1, 2026 - 05:21:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 359 : data loading time 0.00 +[May 1, 2026 - 05:21:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 359 : avg forward pass time 10.88 +[May 1, 2026 - 05:21:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 359 : backward pass time 10.17 +[May 1, 2026 - 05:21:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 359 : optimizer step time 1.58 +[May 1, 2026 - 05:21:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 359-------------------- +[May 1, 2026 - 05:21:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 05:21:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 05:21:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:21:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 05:24:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452196 41.608841 41.323322 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.052246 136.070312 135.998047 +util 62.000000 67.000000 58.000000 +[May 1, 2026 - 05:24:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 360 : avg iteration time 166.26 seconds +[May 1, 2026 - 05:24:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 360 : data loading time 0.00 +[May 1, 2026 - 05:24:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 360 : avg forward pass time 74.52 +[May 1, 2026 - 05:24:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 360 : backward pass time 8.44 +[May 1, 2026 - 05:24:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 360 : optimizer step time 0.39 +[May 1, 2026 - 05:24:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 360-------------------- +[May 1, 2026 - 05:24:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 320802.2256 iter count: 1.0 +[May 1, 2026 - 05:24:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5639 iter count: 1.0 +[May 1, 2026 - 05:24:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 320802.2256iter count: 1.0 +[May 1, 2026 - 05:24:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:24:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.7550iter count: 1.0 +[May 1, 2026 - 05:24:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.6260 iter count: 1.0 +[May 1, 2026 - 05:24:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.0927 iter count: 1.0 +[May 1, 2026 - 05:24:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 579732.9375iter count: 1.0 +[May 1, 2026 - 05:24:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 3.4977 iter count: 1.0 +[May 1, 2026 - 05:24:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 10696393.0000iter count: 1.0 +[May 1, 2026 - 05:24:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5639iter count: 1.0 +[May 1, 2026 - 05:24:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 8.0927 iter count: 1.0 +[May 1, 2026 - 05:24:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 10696393.0000iter count: 1.0 +[May 1, 2026 - 05:24:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.1094 iter count: 1.0 +[May 1, 2026 - 05:24:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 3709.7419iter count: 1.0 +[May 1, 2026 - 05:24:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 7.4060 iter count: 1.0 +[May 1, 2026 - 05:24:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 2708705.0000iter count: 1.0 +[May 1, 2026 - 05:24:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.5205 iter count: 1.0 +[May 1, 2026 - 05:24:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 460965.5312iter count: 1.0 +[May 1, 2026 - 05:24:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 05:24:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:24:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 05:25:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452259 41.609093 41.323322 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.876465 67.992188 67.529297 +util 93.250000 99.000000 87.000000 +[May 1, 2026 - 05:25:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 361 : avg iteration time 47.89 seconds +[May 1, 2026 - 05:25:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 361 : data loading time 0.00 +[May 1, 2026 - 05:25:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 361 : avg forward pass time 13.02 +[May 1, 2026 - 05:25:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 361 : backward pass time 10.13 +[May 1, 2026 - 05:25:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 361 : optimizer step time 1.57 +[May 1, 2026 - 05:25:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 361-------------------- +[May 1, 2026 - 05:25:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0111 iter count: 1.0 +[May 1, 2026 - 05:25:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 1, 2026 - 05:25:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:25:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 05:25:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452576 41.610352 41.323322 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.842285 67.953125 67.509766 +util 98.000000 100.000000 96.000000 +[May 1, 2026 - 05:25:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 362 : avg iteration time 43.58 seconds +[May 1, 2026 - 05:25:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 362 : data loading time 0.00 +[May 1, 2026 - 05:25:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 362 : avg forward pass time 10.86 +[May 1, 2026 - 05:25:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 362 : backward pass time 10.12 +[May 1, 2026 - 05:25:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 362 : optimizer step time 1.57 +[May 1, 2026 - 05:25:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 362-------------------- +[May 1, 2026 - 05:25:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 05:25:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 05:25:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:25:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 05:26:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452591 41.610348 41.323364 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.764160 67.855469 67.490234 +util 95.750000 100.000000 91.000000 +[May 1, 2026 - 05:26:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 363 : avg iteration time 46.26 seconds +[May 1, 2026 - 05:26:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 363 : data loading time 0.00 +[May 1, 2026 - 05:26:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 363 : avg forward pass time 12.21 +[May 1, 2026 - 05:26:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 363 : backward pass time 10.13 +[May 1, 2026 - 05:26:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 363 : optimizer step time 1.56 +[May 1, 2026 - 05:26:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 363-------------------- +[May 1, 2026 - 05:26:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 05:26:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 05:26:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:26:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 05:27:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452585 41.610348 41.323364 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.925293 68.070312 67.490234 +util 94.250000 99.000000 89.000000 +[May 1, 2026 - 05:27:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 364 : avg iteration time 43.58 seconds +[May 1, 2026 - 05:27:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 364 : data loading time 0.00 +[May 1, 2026 - 05:27:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 364 : avg forward pass time 10.86 +[May 1, 2026 - 05:27:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 364 : backward pass time 10.13 +[May 1, 2026 - 05:27:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 364 : optimizer step time 1.57 +[May 1, 2026 - 05:27:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 364-------------------- +[May 1, 2026 - 05:27:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[May 1, 2026 - 05:27:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[May 1, 2026 - 05:27:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:27:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 05:30:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.451991 41.607903 41.323387 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.037598 136.056641 136.031250 +util 67.500000 72.000000 64.000000 +[May 1, 2026 - 05:30:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 365 : avg iteration time 165.55 seconds +[May 1, 2026 - 05:30:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 365 : data loading time 0.00 +[May 1, 2026 - 05:30:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 365 : avg forward pass time 74.17 +[May 1, 2026 - 05:30:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 365 : backward pass time 8.41 +[May 1, 2026 - 05:30:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 365 : optimizer step time 0.40 +[May 1, 2026 - 05:30:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 365-------------------- +[May 1, 2026 - 05:30:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 689682.0469 iter count: 1.0 +[May 1, 2026 - 05:30:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6123 iter count: 1.0 +[May 1, 2026 - 05:30:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 689682.0469iter count: 1.0 +[May 1, 2026 - 05:30:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:30:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 5.2788iter count: 1.0 +[May 1, 2026 - 05:30:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.4242 iter count: 1.0 +[May 1, 2026 - 05:30:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.0723 iter count: 1.0 +[May 1, 2026 - 05:30:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 1085331.5000iter count: 1.0 +[May 1, 2026 - 05:30:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 17.2617iter count: 1.0 +[May 1, 2026 - 05:30:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 10268195.0000iter count: 1.0 +[May 1, 2026 - 05:30:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6123iter count: 1.0 +[May 1, 2026 - 05:30:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.6751 iter count: 1.0 +[May 1, 2026 - 05:30:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 84982.3438iter count: 1.0 +[May 1, 2026 - 05:30:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 6.1015 iter count: 1.0 +[May 1, 2026 - 05:30:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 199375.0156iter count: 1.0 +[May 1, 2026 - 05:30:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 6.1879 iter count: 1.0 +[May 1, 2026 - 05:30:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 236987.1094iter count: 1.0 +[May 1, 2026 - 05:30:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.3061 iter count: 1.0 +[May 1, 2026 - 05:30:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 40630.3008iter count: 1.0 +[May 1, 2026 - 05:30:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 1, 2026 - 05:30:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:30:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 05:30:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.451991 41.607903 41.323387 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.691406 67.757812 67.492188 +util 98.250000 100.000000 96.000000 +[May 1, 2026 - 05:30:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 366 : avg iteration time 48.01 seconds +[May 1, 2026 - 05:30:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 366 : data loading time 0.00 +[May 1, 2026 - 05:30:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 366 : avg forward pass time 13.08 +[May 1, 2026 - 05:30:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 366 : backward pass time 10.14 +[May 1, 2026 - 05:30:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 366 : optimizer step time 1.57 +[May 1, 2026 - 05:30:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 366-------------------- +[May 1, 2026 - 05:30:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 05:30:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 05:30:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:30:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 05:31:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452176 41.608665 41.323387 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.695801 67.796875 67.392578 +util 95.250000 100.000000 90.000000 +[May 1, 2026 - 05:31:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 367 : avg iteration time 46.26 seconds +[May 1, 2026 - 05:31:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 367 : data loading time 0.00 +[May 1, 2026 - 05:31:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 367 : avg forward pass time 12.20 +[May 1, 2026 - 05:31:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 367 : backward pass time 10.15 +[May 1, 2026 - 05:31:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 367 : optimizer step time 1.56 +[May 1, 2026 - 05:31:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 367-------------------- +[May 1, 2026 - 05:31:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0100 iter count: 1.0 +[May 1, 2026 - 05:31:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 1, 2026 - 05:31:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:31:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 05:32:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452171 41.608646 41.323387 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.822754 67.914062 67.548828 +util 96.250000 99.000000 93.000000 +[May 1, 2026 - 05:32:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 368 : avg iteration time 49.10 seconds +[May 1, 2026 - 05:32:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 368 : data loading time 0.00 +[May 1, 2026 - 05:32:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 368 : avg forward pass time 13.61 +[May 1, 2026 - 05:32:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 368 : backward pass time 10.16 +[May 1, 2026 - 05:32:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 368 : optimizer step time 1.55 +[May 1, 2026 - 05:32:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 368-------------------- +[May 1, 2026 - 05:32:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 1, 2026 - 05:32:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 1, 2026 - 05:32:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:32:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 05:33:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452188 41.608646 41.323406 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.798340 67.933594 67.392578 +util 92.500000 96.000000 89.000000 +[May 1, 2026 - 05:33:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 369 : avg iteration time 48.95 seconds +[May 1, 2026 - 05:33:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 369 : data loading time 0.00 +[May 1, 2026 - 05:33:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 369 : avg forward pass time 13.55 +[May 1, 2026 - 05:33:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 369 : backward pass time 10.12 +[May 1, 2026 - 05:33:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 369 : optimizer step time 1.58 +[May 1, 2026 - 05:33:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 369-------------------- +[May 1, 2026 - 05:33:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0112 iter count: 1.0 +[May 1, 2026 - 05:33:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 1, 2026 - 05:33:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:33:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 05:36:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.451866 41.607399 41.323410 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 135.987305 135.990234 135.978516 +util 69.000000 73.000000 65.000000 +[May 1, 2026 - 05:36:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 370 : avg iteration time 168.90 seconds +[May 1, 2026 - 05:36:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 370 : data loading time 0.00 +[May 1, 2026 - 05:36:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 370 : avg forward pass time 75.86 +[May 1, 2026 - 05:36:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 370 : backward pass time 8.38 +[May 1, 2026 - 05:36:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 370 : optimizer step time 0.40 +[May 1, 2026 - 05:36:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 370-------------------- +[May 1, 2026 - 05:36:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 303721.3760 iter count: 1.0 +[May 1, 2026 - 05:36:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5880 iter count: 1.0 +[May 1, 2026 - 05:36:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 303721.3760iter count: 1.0 +[May 1, 2026 - 05:36:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:36:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.5371iter count: 1.0 +[May 1, 2026 - 05:36:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.4098 iter count: 1.0 +[May 1, 2026 - 05:36:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.7764 iter count: 1.0 +[May 1, 2026 - 05:36:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 498968.1250iter count: 1.0 +[May 1, 2026 - 05:36:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 2.2695 iter count: 1.0 +[May 1, 2026 - 05:36:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 5681732.5000iter count: 1.0 +[May 1, 2026 - 05:36:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5880iter count: 1.0 +[May 1, 2026 - 05:36:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 2.5993 iter count: 1.0 +[May 1, 2026 - 05:36:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 181.0367iter count: 1.0 +[May 1, 2026 - 05:36:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 7.2020 iter count: 1.0 +[May 1, 2026 - 05:36:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 1801379.3750iter count: 1.0 +[May 1, 2026 - 05:36:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.5501 iter count: 1.0 +[May 1, 2026 - 05:36:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 1212.1478iter count: 1.0 +[May 1, 2026 - 05:36:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.6492 iter count: 1.0 +[May 1, 2026 - 05:36:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 596239.6875iter count: 1.0 +[May 1, 2026 - 05:36:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 05:36:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:36:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 05:36:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.451999 41.607906 41.323410 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.696289 67.777344 67.453125 +util 94.750000 99.000000 91.000000 +[May 1, 2026 - 05:36:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 371 : avg iteration time 45.13 seconds +[May 1, 2026 - 05:36:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 371 : data loading time 0.00 +[May 1, 2026 - 05:36:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 371 : avg forward pass time 11.64 +[May 1, 2026 - 05:36:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 371 : backward pass time 10.15 +[May 1, 2026 - 05:36:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 371 : optimizer step time 1.57 +[May 1, 2026 - 05:36:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 371-------------------- +[May 1, 2026 - 05:36:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0126 iter count: 1.0 +[May 1, 2026 - 05:36:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0126 iter count: 1.0 +[May 1, 2026 - 05:36:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:36:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 05:37:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452360 41.609398 41.323387 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.730469 67.835938 67.414062 +util 92.250000 96.000000 88.000000 +[May 1, 2026 - 05:37:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 372 : avg iteration time 43.57 seconds +[May 1, 2026 - 05:37:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 372 : data loading time 0.00 +[May 1, 2026 - 05:37:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 372 : avg forward pass time 10.87 +[May 1, 2026 - 05:37:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 372 : backward pass time 10.12 +[May 1, 2026 - 05:37:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 372 : optimizer step time 1.56 +[May 1, 2026 - 05:37:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 372-------------------- +[May 1, 2026 - 05:37:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 05:37:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 05:37:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:37:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 05:38:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452430 41.609650 41.323391 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.813477 67.933594 67.453125 +util 94.000000 99.000000 89.000000 +[May 1, 2026 - 05:38:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 373 : avg iteration time 48.98 seconds +[May 1, 2026 - 05:38:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 373 : data loading time 0.00 +[May 1, 2026 - 05:38:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 373 : avg forward pass time 13.58 +[May 1, 2026 - 05:38:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 373 : backward pass time 10.12 +[May 1, 2026 - 05:38:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 373 : optimizer step time 1.55 +[May 1, 2026 - 05:38:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 373-------------------- +[May 1, 2026 - 05:38:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0114 iter count: 1.0 +[May 1, 2026 - 05:38:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 1, 2026 - 05:38:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:38:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 05:39:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452307 41.609184 41.323387 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.750000 67.835938 67.492188 +util 95.250000 100.000000 90.000000 +[May 1, 2026 - 05:39:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 374 : avg iteration time 46.36 seconds +[May 1, 2026 - 05:39:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 374 : data loading time 0.00 +[May 1, 2026 - 05:39:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 374 : avg forward pass time 12.25 +[May 1, 2026 - 05:39:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 374 : backward pass time 10.14 +[May 1, 2026 - 05:39:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 374 : optimizer step time 1.57 +[May 1, 2026 - 05:39:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 374-------------------- +[May 1, 2026 - 05:39:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0114 iter count: 1.0 +[May 1, 2026 - 05:39:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 1, 2026 - 05:39:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:39:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 05:41:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452261 41.608883 41.323418 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.045898 136.068359 135.978516 +util 62.750000 67.000000 59.000000 +[May 1, 2026 - 05:41:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 375 : avg iteration time 166.28 seconds +[May 1, 2026 - 05:41:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 375 : data loading time 0.00 +[May 1, 2026 - 05:41:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 375 : avg forward pass time 74.54 +[May 1, 2026 - 05:41:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 375 : backward pass time 8.43 +[May 1, 2026 - 05:41:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 375 : optimizer step time 0.40 +[May 1, 2026 - 05:41:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 375-------------------- +[May 1, 2026 - 05:41:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 1085287.3809 iter count: 1.0 +[May 1, 2026 - 05:41:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5876 iter count: 1.0 +[May 1, 2026 - 05:41:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 1085287.3809iter count: 1.0 +[May 1, 2026 - 05:41:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:41:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.5954iter count: 1.0 +[May 1, 2026 - 05:41:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.1103 iter count: 1.0 +[May 1, 2026 - 05:41:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.6924 iter count: 1.0 +[May 1, 2026 - 05:41:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 2125446.5000iter count: 1.0 +[May 1, 2026 - 05:41:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 9.2135 iter count: 1.0 +[May 1, 2026 - 05:41:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 35488588.0000iter count: 1.0 +[May 1, 2026 - 05:41:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5876iter count: 1.0 +[May 1, 2026 - 05:41:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.0948 iter count: 1.0 +[May 1, 2026 - 05:41:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 3603.5762iter count: 1.0 +[May 1, 2026 - 05:41:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 6.3443 iter count: 1.0 +[May 1, 2026 - 05:41:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 324031.1250iter count: 1.0 +[May 1, 2026 - 05:41:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.7271 iter count: 1.0 +[May 1, 2026 - 05:41:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 1727.2413iter count: 1.0 +[May 1, 2026 - 05:41:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.9932 iter count: 1.0 +[May 1, 2026 - 05:41:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 21727.1641iter count: 1.0 +[May 1, 2026 - 05:41:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 05:41:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:41:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 05:42:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452282 41.608883 41.323463 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.803711 67.894531 67.531250 +util 94.500000 100.000000 89.000000 +[May 1, 2026 - 05:42:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 376 : avg iteration time 47.71 seconds +[May 1, 2026 - 05:42:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 376 : data loading time 0.00 +[May 1, 2026 - 05:42:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 376 : avg forward pass time 12.93 +[May 1, 2026 - 05:42:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 376 : backward pass time 10.12 +[May 1, 2026 - 05:42:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 376 : optimizer step time 1.58 +[May 1, 2026 - 05:42:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 376-------------------- +[May 1, 2026 - 05:42:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 05:42:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 05:42:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:42:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 05:43:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452468 41.609638 41.323463 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.867188 67.933594 67.667969 +util 93.500000 97.000000 90.000000 +[May 1, 2026 - 05:43:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 377 : avg iteration time 46.29 seconds +[May 1, 2026 - 05:43:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 377 : data loading time 0.00 +[May 1, 2026 - 05:43:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 377 : avg forward pass time 12.20 +[May 1, 2026 - 05:43:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 377 : backward pass time 10.15 +[May 1, 2026 - 05:43:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 377 : optimizer step time 1.58 +[May 1, 2026 - 05:43:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 377-------------------- +[May 1, 2026 - 05:43:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0112 iter count: 1.0 +[May 1, 2026 - 05:43:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 1, 2026 - 05:43:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:43:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 05:44:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452472 41.609638 41.323471 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.833008 67.933594 67.531250 +util 92.500000 99.000000 86.000000 +[May 1, 2026 - 05:44:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 378 : avg iteration time 46.29 seconds +[May 1, 2026 - 05:44:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 378 : data loading time 0.00 +[May 1, 2026 - 05:44:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 378 : avg forward pass time 12.23 +[May 1, 2026 - 05:44:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 378 : backward pass time 10.12 +[May 1, 2026 - 05:44:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 378 : optimizer step time 1.58 +[May 1, 2026 - 05:44:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 378-------------------- +[May 1, 2026 - 05:44:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 05:44:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 05:44:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:44:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 05:45:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452467 41.609646 41.323463 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.857422 67.933594 67.628906 +util 97.500000 100.000000 94.000000 +[May 1, 2026 - 05:45:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 379 : avg iteration time 46.16 seconds +[May 1, 2026 - 05:45:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 379 : data loading time 0.00 +[May 1, 2026 - 05:45:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 379 : avg forward pass time 12.16 +[May 1, 2026 - 05:45:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 379 : backward pass time 10.13 +[May 1, 2026 - 05:45:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 379 : optimizer step time 1.58 +[May 1, 2026 - 05:45:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 379-------------------- +[May 1, 2026 - 05:45:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0104 iter count: 1.0 +[May 1, 2026 - 05:45:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 1, 2026 - 05:45:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:45:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 05:47:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452104 41.607525 41.323685 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.011719 136.017578 136.009766 +util 62.000000 66.000000 58.000000 +[May 1, 2026 - 05:47:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 380 : avg iteration time 165.95 seconds +[May 1, 2026 - 05:47:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 380 : data loading time 0.00 +[May 1, 2026 - 05:47:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 380 : avg forward pass time 74.38 +[May 1, 2026 - 05:47:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 380 : backward pass time 8.40 +[May 1, 2026 - 05:47:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 380 : optimizer step time 0.40 +[May 1, 2026 - 05:47:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 380-------------------- +[May 1, 2026 - 05:47:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 160949.3936 iter count: 1.0 +[May 1, 2026 - 05:47:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5742 iter count: 1.0 +[May 1, 2026 - 05:47:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 160949.3936iter count: 1.0 +[May 1, 2026 - 05:47:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:47:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.6476iter count: 1.0 +[May 1, 2026 - 05:47:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.9153 iter count: 1.0 +[May 1, 2026 - 05:47:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.6321 iter count: 1.0 +[May 1, 2026 - 05:47:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 308867.7812iter count: 1.0 +[May 1, 2026 - 05:47:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 46.0865iter count: 1.0 +[May 1, 2026 - 05:47:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 4257576.5000iter count: 1.0 +[May 1, 2026 - 05:47:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5742iter count: 1.0 +[May 1, 2026 - 05:47:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.9358 iter count: 1.0 +[May 1, 2026 - 05:47:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 19371.4824iter count: 1.0 +[May 1, 2026 - 05:47:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.8320 iter count: 1.0 +[May 1, 2026 - 05:47:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 116299.4141iter count: 1.0 +[May 1, 2026 - 05:47:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.3945 iter count: 1.0 +[May 1, 2026 - 05:47:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 6561.5269iter count: 1.0 +[May 1, 2026 - 05:47:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.4530 iter count: 1.0 +[May 1, 2026 - 05:47:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 998.3111iter count: 1.0 +[May 1, 2026 - 05:47:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 05:47:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:47:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 05:48:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452298 41.608284 41.323685 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.798340 67.933594 67.392578 +util 96.500000 100.000000 93.000000 +[May 1, 2026 - 05:48:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 381 : avg iteration time 47.84 seconds +[May 1, 2026 - 05:48:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 381 : data loading time 0.00 +[May 1, 2026 - 05:48:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 381 : avg forward pass time 12.98 +[May 1, 2026 - 05:48:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 381 : backward pass time 10.16 +[May 1, 2026 - 05:48:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 381 : optimizer step time 1.58 +[May 1, 2026 - 05:48:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 381-------------------- +[May 1, 2026 - 05:48:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0114 iter count: 1.0 +[May 1, 2026 - 05:48:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 1, 2026 - 05:48:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:48:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 05:49:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452683 41.609776 41.323734 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.749512 67.816406 67.548828 +util 93.750000 98.000000 90.000000 +[May 1, 2026 - 05:49:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 382 : avg iteration time 48.96 seconds +[May 1, 2026 - 05:49:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 382 : data loading time 0.00 +[May 1, 2026 - 05:49:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 382 : avg forward pass time 13.55 +[May 1, 2026 - 05:49:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 382 : backward pass time 10.14 +[May 1, 2026 - 05:49:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 382 : optimizer step time 1.57 +[May 1, 2026 - 05:49:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 382-------------------- +[May 1, 2026 - 05:49:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 05:49:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 05:49:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:49:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 05:50:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452673 41.609776 41.323689 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.817871 67.953125 67.412109 +util 94.250000 98.000000 90.000000 +[May 1, 2026 - 05:50:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 383 : avg iteration time 43.60 seconds +[May 1, 2026 - 05:50:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 383 : data loading time 0.00 +[May 1, 2026 - 05:50:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 383 : avg forward pass time 10.87 +[May 1, 2026 - 05:50:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 383 : backward pass time 10.14 +[May 1, 2026 - 05:50:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 383 : optimizer step time 1.56 +[May 1, 2026 - 05:50:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 383-------------------- +[May 1, 2026 - 05:50:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 05:50:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 05:50:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:50:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 05:50:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452508 41.609104 41.323692 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.905762 67.953125 67.763672 +util 93.250000 97.000000 89.000000 +[May 1, 2026 - 05:50:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 384 : avg iteration time 49.18 seconds +[May 1, 2026 - 05:50:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 384 : data loading time 0.00 +[May 1, 2026 - 05:50:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 384 : avg forward pass time 13.66 +[May 1, 2026 - 05:50:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 384 : backward pass time 10.16 +[May 1, 2026 - 05:50:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 384 : optimizer step time 1.57 +[May 1, 2026 - 05:50:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 384-------------------- +[May 1, 2026 - 05:50:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 05:50:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 05:50:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:50:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 05:53:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452329 41.608387 41.323692 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.084961 136.107422 136.017578 +util 60.000000 64.000000 56.000000 +[May 1, 2026 - 05:53:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 385 : avg iteration time 165.98 seconds +[May 1, 2026 - 05:53:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 385 : data loading time 0.00 +[May 1, 2026 - 05:53:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 385 : avg forward pass time 74.39 +[May 1, 2026 - 05:53:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 385 : backward pass time 8.41 +[May 1, 2026 - 05:53:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 385 : optimizer step time 0.39 +[May 1, 2026 - 05:53:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 385-------------------- +[May 1, 2026 - 05:53:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 135934.9004 iter count: 1.0 +[May 1, 2026 - 05:53:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5842 iter count: 1.0 +[May 1, 2026 - 05:53:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 135934.9004iter count: 1.0 +[May 1, 2026 - 05:53:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:53:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.6635iter count: 1.0 +[May 1, 2026 - 05:53:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 2.4037 iter count: 1.0 +[May 1, 2026 - 05:53:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.5086 iter count: 1.0 +[May 1, 2026 - 05:53:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 257493.5312iter count: 1.0 +[May 1, 2026 - 05:53:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 122.4141iter count: 1.0 +[May 1, 2026 - 05:53:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 3325570.7500iter count: 1.0 +[May 1, 2026 - 05:53:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5842iter count: 1.0 +[May 1, 2026 - 05:53:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.2628 iter count: 1.0 +[May 1, 2026 - 05:53:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 682.3777iter count: 1.0 +[May 1, 2026 - 05:53:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.1537 iter count: 1.0 +[May 1, 2026 - 05:53:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 548.6448iter count: 1.0 +[May 1, 2026 - 05:53:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 6.1200 iter count: 1.0 +[May 1, 2026 - 05:53:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 206901.8594iter count: 1.0 +[May 1, 2026 - 05:53:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.2817 iter count: 1.0 +[May 1, 2026 - 05:53:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 5236.1865iter count: 1.0 +[May 1, 2026 - 05:53:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 05:53:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:53:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 05:54:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452330 41.608368 41.323715 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.852051 67.992188 67.431641 +util 95.750000 100.000000 91.000000 +[May 1, 2026 - 05:54:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 386 : avg iteration time 50.57 seconds +[May 1, 2026 - 05:54:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 386 : data loading time 0.00 +[May 1, 2026 - 05:54:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 386 : avg forward pass time 14.36 +[May 1, 2026 - 05:54:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 386 : backward pass time 10.14 +[May 1, 2026 - 05:54:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 386 : optimizer step time 1.58 +[May 1, 2026 - 05:54:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 386-------------------- +[May 1, 2026 - 05:54:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0117 iter count: 1.0 +[May 1, 2026 - 05:54:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[May 1, 2026 - 05:54:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:54:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 05:55:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452584 41.609371 41.323730 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.740234 67.855469 67.394531 +util 94.250000 100.000000 88.000000 +[May 1, 2026 - 05:55:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 387 : avg iteration time 46.37 seconds +[May 1, 2026 - 05:55:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 387 : data loading time 0.00 +[May 1, 2026 - 05:55:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 387 : avg forward pass time 12.25 +[May 1, 2026 - 05:55:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 387 : backward pass time 10.15 +[May 1, 2026 - 05:55:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 387 : optimizer step time 1.58 +[May 1, 2026 - 05:55:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 387-------------------- +[May 1, 2026 - 05:55:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 1, 2026 - 05:55:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 1, 2026 - 05:55:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:55:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 05:56:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452641 41.609627 41.323692 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.779297 67.875000 67.492188 +util 94.250000 98.000000 90.000000 +[May 1, 2026 - 05:56:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 388 : avg iteration time 46.30 seconds +[May 1, 2026 - 05:56:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 388 : data loading time 0.00 +[May 1, 2026 - 05:56:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 388 : avg forward pass time 12.22 +[May 1, 2026 - 05:56:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 388 : backward pass time 10.15 +[May 1, 2026 - 05:56:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 388 : optimizer step time 1.56 +[May 1, 2026 - 05:56:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 388-------------------- +[May 1, 2026 - 05:56:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 05:56:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 05:56:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:56:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 05:56:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452559 41.609306 41.323692 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.774414 67.894531 67.414062 +util 96.500000 99.000000 93.000000 +[May 1, 2026 - 05:56:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 389 : avg iteration time 49.04 seconds +[May 1, 2026 - 05:56:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 389 : data loading time 0.00 +[May 1, 2026 - 05:56:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 389 : avg forward pass time 13.59 +[May 1, 2026 - 05:56:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 389 : backward pass time 10.13 +[May 1, 2026 - 05:56:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 389 : optimizer step time 1.58 +[May 1, 2026 - 05:56:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 389-------------------- +[May 1, 2026 - 05:56:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 05:56:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 05:56:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:56:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 05:59:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452424 41.608734 41.323692 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.075195 136.087891 136.037109 +util 64.250000 71.000000 58.000000 +[May 1, 2026 - 05:59:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 390 : avg iteration time 171.28 seconds +[May 1, 2026 - 05:59:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 390 : data loading time 0.00 +[May 1, 2026 - 05:59:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 390 : avg forward pass time 77.02 +[May 1, 2026 - 05:59:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 390 : backward pass time 8.41 +[May 1, 2026 - 05:59:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 390 : optimizer step time 0.40 +[May 1, 2026 - 05:59:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 390-------------------- +[May 1, 2026 - 05:59:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 1658641.9683 iter count: 1.0 +[May 1, 2026 - 05:59:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6122 iter count: 1.0 +[May 1, 2026 - 05:59:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 1658641.9683iter count: 1.0 +[May 1, 2026 - 05:59:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:59:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.7992iter count: 1.0 +[May 1, 2026 - 05:59:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.4670 iter count: 1.0 +[May 1, 2026 - 05:59:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.8439 iter count: 1.0 +[May 1, 2026 - 05:59:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 2351570.0000iter count: 1.0 +[May 1, 2026 - 05:59:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 2.5448 iter count: 1.0 +[May 1, 2026 - 05:59:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 48053716.0000iter count: 1.0 +[May 1, 2026 - 05:59:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6122iter count: 1.0 +[May 1, 2026 - 05:59:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.9304 iter count: 1.0 +[May 1, 2026 - 05:59:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 19162.8223iter count: 1.0 +[May 1, 2026 - 05:59:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.9189 iter count: 1.0 +[May 1, 2026 - 05:59:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 138381.6250iter count: 1.0 +[May 1, 2026 - 05:59:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.1340 iter count: 1.0 +[May 1, 2026 - 05:59:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 527.3710iter count: 1.0 +[May 1, 2026 - 05:59:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.8040 iter count: 1.0 +[May 1, 2026 - 05:59:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 14882.9453iter count: 1.0 +[May 1, 2026 - 05:59:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 05:59:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 05:59:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 06:00:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452430 41.608738 41.323734 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.676758 67.757812 67.433594 +util 92.500000 97.000000 88.000000 +[May 1, 2026 - 06:00:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 391 : avg iteration time 47.86 seconds +[May 1, 2026 - 06:00:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 391 : data loading time 0.00 +[May 1, 2026 - 06:00:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 391 : avg forward pass time 13.00 +[May 1, 2026 - 06:00:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 391 : backward pass time 10.14 +[May 1, 2026 - 06:00:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 391 : optimizer step time 1.58 +[May 1, 2026 - 06:00:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 391-------------------- +[May 1, 2026 - 06:00:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0117 iter count: 1.0 +[May 1, 2026 - 06:00:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[May 1, 2026 - 06:00:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:00:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 06:01:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452606 41.609486 41.323692 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.725586 67.816406 67.453125 +util 92.000000 97.000000 87.000000 +[May 1, 2026 - 06:01:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 392 : avg iteration time 46.31 seconds +[May 1, 2026 - 06:01:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 392 : data loading time 0.00 +[May 1, 2026 - 06:01:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 392 : avg forward pass time 12.22 +[May 1, 2026 - 06:01:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 392 : backward pass time 10.15 +[May 1, 2026 - 06:01:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 392 : optimizer step time 1.59 +[May 1, 2026 - 06:01:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 392-------------------- +[May 1, 2026 - 06:01:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 1, 2026 - 06:01:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 1, 2026 - 06:01:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:01:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 06:02:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452608 41.609486 41.323692 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.862305 67.992188 67.472656 +util 94.500000 99.000000 89.000000 +[May 1, 2026 - 06:02:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 393 : avg iteration time 46.23 seconds +[May 1, 2026 - 06:02:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 393 : data loading time 0.00 +[May 1, 2026 - 06:02:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 393 : avg forward pass time 12.19 +[May 1, 2026 - 06:02:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 393 : backward pass time 10.12 +[May 1, 2026 - 06:02:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 393 : optimizer step time 1.57 +[May 1, 2026 - 06:02:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 393-------------------- +[May 1, 2026 - 06:02:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 06:02:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 06:02:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:02:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 06:02:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.453079 41.611195 41.323692 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.891602 68.031250 67.472656 +util 97.750000 100.000000 95.000000 +[May 1, 2026 - 06:02:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 394 : avg iteration time 43.55 seconds +[May 1, 2026 - 06:02:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 394 : data loading time 0.00 +[May 1, 2026 - 06:02:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 394 : avg forward pass time 10.85 +[May 1, 2026 - 06:02:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 394 : backward pass time 10.14 +[May 1, 2026 - 06:02:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 394 : optimizer step time 1.57 +[May 1, 2026 - 06:02:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 394-------------------- +[May 1, 2026 - 06:02:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 1, 2026 - 06:02:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 1, 2026 - 06:02:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:02:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 06:05:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452930 41.610519 41.323818 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.018066 136.037109 136.011719 +util 62.250000 69.000000 56.000000 +[May 1, 2026 - 06:05:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 395 : avg iteration time 160.44 seconds +[May 1, 2026 - 06:05:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 395 : data loading time 0.00 +[May 1, 2026 - 06:05:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 395 : avg forward pass time 71.64 +[May 1, 2026 - 06:05:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 395 : backward pass time 8.42 +[May 1, 2026 - 06:05:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 395 : optimizer step time 0.40 +[May 1, 2026 - 06:05:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 395-------------------- +[May 1, 2026 - 06:05:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 1378607.9102 iter count: 1.0 +[May 1, 2026 - 06:05:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5516 iter count: 1.0 +[May 1, 2026 - 06:05:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 1378607.9102iter count: 1.0 +[May 1, 2026 - 06:05:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:05:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 5.0864iter count: 1.0 +[May 1, 2026 - 06:05:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.6029 iter count: 1.0 +[May 1, 2026 - 06:05:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.9904 iter count: 1.0 +[May 1, 2026 - 06:05:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 2797821.2500iter count: 1.0 +[May 1, 2026 - 06:05:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 24.6755iter count: 1.0 +[May 1, 2026 - 06:05:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 64406940.0000iter count: 1.0 +[May 1, 2026 - 06:05:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5516iter count: 1.0 +[May 1, 2026 - 06:05:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.1934 iter count: 1.0 +[May 1, 2026 - 06:05:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 593.9567iter count: 1.0 +[May 1, 2026 - 06:05:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.6789 iter count: 1.0 +[May 1, 2026 - 06:05:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 85631.4688iter count: 1.0 +[May 1, 2026 - 06:05:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 6.1999 iter count: 1.0 +[May 1, 2026 - 06:05:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 242730.9062iter count: 1.0 +[May 1, 2026 - 06:05:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.4140 iter count: 1.0 +[May 1, 2026 - 06:05:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 372483.3438iter count: 1.0 +[May 1, 2026 - 06:05:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 06:05:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:05:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 06:06:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452934 41.610527 41.323818 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.686035 67.757812 67.470703 +util 98.000000 100.000000 95.000000 +[May 1, 2026 - 06:06:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 396 : avg iteration time 45.15 seconds +[May 1, 2026 - 06:06:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 396 : data loading time 0.00 +[May 1, 2026 - 06:06:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 396 : avg forward pass time 11.65 +[May 1, 2026 - 06:06:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 396 : backward pass time 10.14 +[May 1, 2026 - 06:06:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 396 : optimizer step time 1.57 +[May 1, 2026 - 06:06:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 396-------------------- +[May 1, 2026 - 06:06:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 1, 2026 - 06:06:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 1, 2026 - 06:06:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:06:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 06:07:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.453038 41.610943 41.323818 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.857422 67.992188 67.453125 +util 95.000000 100.000000 90.000000 +[May 1, 2026 - 06:07:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 397 : avg iteration time 46.26 seconds +[May 1, 2026 - 06:07:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 397 : data loading time 0.00 +[May 1, 2026 - 06:07:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 397 : avg forward pass time 12.21 +[May 1, 2026 - 06:07:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 397 : backward pass time 10.15 +[May 1, 2026 - 06:07:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 397 : optimizer step time 1.57 +[May 1, 2026 - 06:07:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 397-------------------- +[May 1, 2026 - 06:07:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 06:07:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 06:07:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:07:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 06:07:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.453041 41.610943 41.323830 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.886719 67.992188 67.570312 +util 97.500000 100.000000 94.000000 +[May 1, 2026 - 06:07:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 398 : avg iteration time 48.92 seconds +[May 1, 2026 - 06:07:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 398 : data loading time 0.00 +[May 1, 2026 - 06:07:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 398 : avg forward pass time 13.51 +[May 1, 2026 - 06:07:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 398 : backward pass time 10.15 +[May 1, 2026 - 06:07:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 398 : optimizer step time 1.56 +[May 1, 2026 - 06:07:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 398-------------------- +[May 1, 2026 - 06:07:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0111 iter count: 1.0 +[May 1, 2026 - 06:07:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 1, 2026 - 06:07:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:07:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 06:08:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.453085 41.611130 41.323818 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.881836 67.992188 67.550781 +util 96.000000 100.000000 91.000000 +[May 1, 2026 - 06:08:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 399 : avg iteration time 46.29 seconds +[May 1, 2026 - 06:08:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 399 : data loading time 0.00 +[May 1, 2026 - 06:08:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 399 : avg forward pass time 12.21 +[May 1, 2026 - 06:08:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 399 : backward pass time 10.16 +[May 1, 2026 - 06:08:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 399 : optimizer step time 1.56 +[May 1, 2026 - 06:08:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 399-------------------- +[May 1, 2026 - 06:08:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 06:08:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 06:08:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:08:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 06:11:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452624 41.609283 41.323822 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.062012 136.070312 136.037109 +util 64.500000 69.000000 60.000000 +[May 1, 2026 - 06:11:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 400 : avg iteration time 160.88 seconds +[May 1, 2026 - 06:11:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 400 : data loading time 0.00 +[May 1, 2026 - 06:11:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 400 : avg forward pass time 71.84 +[May 1, 2026 - 06:11:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 400 : backward pass time 8.41 +[May 1, 2026 - 06:11:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 400 : optimizer step time 0.39 +[May 1, 2026 - 06:11:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 400-------------------- +[May 1, 2026 - 06:11:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 376234.2266 iter count: 1.0 +[May 1, 2026 - 06:11:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5692 iter count: 1.0 +[May 1, 2026 - 06:11:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 376234.2266iter count: 1.0 +[May 1, 2026 - 06:11:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:11:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 5.3712iter count: 1.0 +[May 1, 2026 - 06:11:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 2.5348 iter count: 1.0 +[May 1, 2026 - 06:11:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.6638 iter count: 1.0 +[May 1, 2026 - 06:11:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 677866.1250iter count: 1.0 +[May 1, 2026 - 06:11:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 159.1263iter count: 1.0 +[May 1, 2026 - 06:11:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 4535747.0000iter count: 1.0 +[May 1, 2026 - 06:11:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5692iter count: 1.0 +[May 1, 2026 - 06:11:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.9978 iter count: 1.0 +[May 1, 2026 - 06:11:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 21930.0488iter count: 1.0 +[May 1, 2026 - 06:11:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.9540 iter count: 1.0 +[May 1, 2026 - 06:11:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 20089.8594iter count: 1.0 +[May 1, 2026 - 06:11:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.8564 iter count: 1.0 +[May 1, 2026 - 06:11:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 122113.1953iter count: 1.0 +[May 1, 2026 - 06:11:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.5684 iter count: 1.0 +[May 1, 2026 - 06:11:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 507296.2188iter count: 1.0 +[May 1, 2026 - 06:11:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 1, 2026 - 06:11:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 1, 2026 - 06:11:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 14.97 seconds +[May 1, 2026 - 06:11:35 | INFO | fastgen.trainer:save_checkpoint:266 ] Saving checkpoint iteration 400 +[May 1, 2026 - 06:11:35 | INFO | fastgen.utils.checkpointer:save:311 ] Saving FSDP model to prefix /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000400 +[May 1, 2026 - 06:12:16 | SUCCESS | fastgen.utils.checkpointer:save:362 ] Model saved at iteration 400 +[May 1, 2026 - 06:12:17 | INFO | fastgen.trainer:validate:358 ] Validation iteration 400 +[May 1, 2026 - 06:12:17 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 06:12:19 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 2.1s +[May 1, 2026 - 06:12:22 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 0: sync_c=6.605 +[May 1, 2026 - 06:12:23 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 06:12:25 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 06:12:28 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 1: sync_c=5.959 +[May 1, 2026 - 06:12:28 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 06:12:30 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 06:12:33 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 2: sync_c=5.342 +[May 1, 2026 - 06:12:33 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 06:12:35 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 06:12:38 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 3: sync_c=5.081 +[May 1, 2026 - 06:12:38 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 06:12:40 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 06:12:43 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 4: sync_c=5.324 +[May 1, 2026 - 06:12:43 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 06:12:45 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 06:12:48 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 5: sync_c=6.905 +[May 1, 2026 - 06:12:48 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 06:12:50 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 06:12:53 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 6: sync_c=6.371 +[May 1, 2026 - 06:12:53 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 06:12:55 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 06:12:58 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 7: sync_c=5.368 +[May 1, 2026 - 06:12:59 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 06:13:00 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 06:13:03 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 8: sync_c=6.093 +[May 1, 2026 - 06:13:04 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 06:13:05 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 06:13:08 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 9: sync_c=3.310 +[May 1, 2026 - 06:13:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging val0 stats at iteration 400-------------------- +[May 1, 2026 - 06:13:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0000 iter count: 10.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 1, 2026 - 06:13:25 | INFO | fastgen.callbacks.wandb:on_validation_end:763 ] Logged 10 val videos at iteration 400 +[May 1, 2026 - 06:13:25 | INFO | fastgen.callbacks.wandb:on_validation_end:769 ] [SyncEval] val0 mean sync_c: 5.636 (n=10) +[May 1, 2026 - 06:14:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.454191 41.617188 41.323280 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.462891 68.820312 68.343750 +util 92.750000 99.000000 87.000000 +[May 1, 2026 - 06:14:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 401 : avg iteration time 170.27 seconds +[May 1, 2026 - 06:14:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 401 : data loading time 0.00 +[May 1, 2026 - 06:14:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 401 : avg forward pass time 11.87 +[May 1, 2026 - 06:14:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 401 : backward pass time 10.17 +[May 1, 2026 - 06:14:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 401 : optimizer step time 1.56 +[May 1, 2026 - 06:14:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 401-------------------- +[May 1, 2026 - 06:14:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0131 iter count: 1.0 +[May 1, 2026 - 06:14:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0131 iter count: 1.0 +[May 1, 2026 - 06:14:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:14:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 06:14:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.454203 41.617188 41.323280 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.487305 68.917969 68.343750 +util 94.750000 100.000000 89.000000 +[May 1, 2026 - 06:14:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 402 : avg iteration time 43.83 seconds +[May 1, 2026 - 06:14:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 402 : data loading time 0.00 +[May 1, 2026 - 06:14:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 402 : avg forward pass time 10.93 +[May 1, 2026 - 06:14:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 402 : backward pass time 10.20 +[May 1, 2026 - 06:14:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 402 : optimizer step time 1.57 +[May 1, 2026 - 06:14:54 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 402-------------------- +[May 1, 2026 - 06:14:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0119 iter count: 1.0 +[May 1, 2026 - 06:14:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[May 1, 2026 - 06:14:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:14:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 06:15:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.453781 41.615547 41.323280 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.487305 68.917969 68.343750 +util 95.000000 100.000000 90.000000 +[May 1, 2026 - 06:15:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 403 : avg iteration time 46.41 seconds +[May 1, 2026 - 06:15:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 403 : data loading time 0.00 +[May 1, 2026 - 06:15:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 403 : avg forward pass time 12.25 +[May 1, 2026 - 06:15:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 403 : backward pass time 10.15 +[May 1, 2026 - 06:15:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 403 : optimizer step time 1.57 +[May 1, 2026 - 06:15:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 403-------------------- +[May 1, 2026 - 06:15:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0122 iter count: 1.0 +[May 1, 2026 - 06:15:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0122 iter count: 1.0 +[May 1, 2026 - 06:15:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:15:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 06:16:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.453788 41.615547 41.323307 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.487305 68.917969 68.343750 +util 97.750000 100.000000 95.000000 +[May 1, 2026 - 06:16:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 404 : avg iteration time 46.34 seconds +[May 1, 2026 - 06:16:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 404 : data loading time 0.00 +[May 1, 2026 - 06:16:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 404 : avg forward pass time 12.23 +[May 1, 2026 - 06:16:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 404 : backward pass time 10.16 +[May 1, 2026 - 06:16:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 404 : optimizer step time 1.57 +[May 1, 2026 - 06:16:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 404-------------------- +[May 1, 2026 - 06:16:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 06:16:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 06:16:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:16:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 06:19:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.453000 41.611046 41.323757 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.016602 136.037109 136.009766 +util 61.250000 66.000000 57.000000 +[May 1, 2026 - 06:19:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 405 : avg iteration time 168.93 seconds +[May 1, 2026 - 06:19:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 405 : data loading time 0.00 +[May 1, 2026 - 06:19:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 405 : avg forward pass time 75.82 +[May 1, 2026 - 06:19:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 405 : backward pass time 8.46 +[May 1, 2026 - 06:19:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 405 : optimizer step time 0.39 +[May 1, 2026 - 06:19:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 405-------------------- +[May 1, 2026 - 06:19:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 1625053.3594 iter count: 1.0 +[May 1, 2026 - 06:19:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5833 iter count: 1.0 +[May 1, 2026 - 06:19:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 1625053.3594iter count: 1.0 +[May 1, 2026 - 06:19:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:19:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.6249iter count: 1.0 +[May 1, 2026 - 06:19:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.6611 iter count: 1.0 +[May 1, 2026 - 06:19:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 9.0759 iter count: 1.0 +[May 1, 2026 - 06:19:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 2583924.7500iter count: 1.0 +[May 1, 2026 - 06:19:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 27.7198iter count: 1.0 +[May 1, 2026 - 06:19:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 76424472.0000iter count: 1.0 +[May 1, 2026 - 06:19:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5833iter count: 1.0 +[May 1, 2026 - 06:19:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 2.2829 iter count: 1.0 +[May 1, 2026 - 06:19:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 96.1429 iter count: 1.0 +[May 1, 2026 - 06:19:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 6.4020 iter count: 1.0 +[May 1, 2026 - 06:19:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 363671.9375iter count: 1.0 +[May 1, 2026 - 06:19:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.3601 iter count: 1.0 +[May 1, 2026 - 06:19:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 6125.9624iter count: 1.0 +[May 1, 2026 - 06:19:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.4887 iter count: 1.0 +[May 1, 2026 - 06:19:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 58531.9141iter count: 1.0 +[May 1, 2026 - 06:19:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 1, 2026 - 06:19:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:19:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 06:20:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.453098 41.611439 41.323757 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.955078 68.011719 67.785156 +util 93.000000 99.000000 88.000000 +[May 1, 2026 - 06:20:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 406 : avg iteration time 47.90 seconds +[May 1, 2026 - 06:20:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 406 : data loading time 0.00 +[May 1, 2026 - 06:20:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 406 : avg forward pass time 13.02 +[May 1, 2026 - 06:20:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 406 : backward pass time 10.13 +[May 1, 2026 - 06:20:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 406 : optimizer step time 1.57 +[May 1, 2026 - 06:20:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 406-------------------- +[May 1, 2026 - 06:20:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0113 iter count: 1.0 +[May 1, 2026 - 06:20:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 1, 2026 - 06:20:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:20:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 06:20:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.453346 41.612381 41.323761 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.955078 68.011719 67.785156 +util 93.250000 100.000000 87.000000 +[May 1, 2026 - 06:20:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 407 : avg iteration time 46.21 seconds +[May 1, 2026 - 06:20:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 407 : data loading time 0.00 +[May 1, 2026 - 06:20:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 407 : avg forward pass time 12.18 +[May 1, 2026 - 06:20:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 407 : backward pass time 10.13 +[May 1, 2026 - 06:20:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 407 : optimizer step time 1.56 +[May 1, 2026 - 06:20:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 407-------------------- +[May 1, 2026 - 06:20:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 1, 2026 - 06:20:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 1, 2026 - 06:20:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:20:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 06:21:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.453341 41.612381 41.323761 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.867188 67.960938 67.835938 +util 93.000000 99.000000 87.000000 +[May 1, 2026 - 06:21:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 408 : avg iteration time 43.65 seconds +[May 1, 2026 - 06:21:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 408 : data loading time 0.00 +[May 1, 2026 - 06:21:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 408 : avg forward pass time 10.90 +[May 1, 2026 - 06:21:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 408 : backward pass time 10.13 +[May 1, 2026 - 06:21:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 408 : optimizer step time 1.56 +[May 1, 2026 - 06:21:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 408-------------------- +[May 1, 2026 - 06:21:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 1, 2026 - 06:21:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 1, 2026 - 06:21:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:21:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 06:22:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.453337 41.612381 41.323761 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.008789 68.031250 67.941406 +util 96.750000 100.000000 93.000000 +[May 1, 2026 - 06:22:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 409 : avg iteration time 46.29 seconds +[May 1, 2026 - 06:22:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 409 : data loading time 0.00 +[May 1, 2026 - 06:22:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 409 : avg forward pass time 12.21 +[May 1, 2026 - 06:22:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 409 : backward pass time 10.15 +[May 1, 2026 - 06:22:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 409 : optimizer step time 1.59 +[May 1, 2026 - 06:22:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 409-------------------- +[May 1, 2026 - 06:22:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0112 iter count: 1.0 +[May 1, 2026 - 06:22:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 1, 2026 - 06:22:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:22:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 06:25:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452684 41.609772 41.323765 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.037598 136.056641 136.031250 +util 62.500000 67.000000 58.000000 +[May 1, 2026 - 06:25:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 410 : avg iteration time 166.15 seconds +[May 1, 2026 - 06:25:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 410 : data loading time 0.00 +[May 1, 2026 - 06:25:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 410 : avg forward pass time 74.48 +[May 1, 2026 - 06:25:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 410 : backward pass time 8.44 +[May 1, 2026 - 06:25:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 410 : optimizer step time 0.39 +[May 1, 2026 - 06:25:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 410-------------------- +[May 1, 2026 - 06:25:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 102551.0068 iter count: 1.0 +[May 1, 2026 - 06:25:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5235 iter count: 1.0 +[May 1, 2026 - 06:25:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 102551.0068iter count: 1.0 +[May 1, 2026 - 06:25:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:25:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.2699iter count: 1.0 +[May 1, 2026 - 06:25:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.0117 iter count: 1.0 +[May 1, 2026 - 06:25:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.2907 iter count: 1.0 +[May 1, 2026 - 06:25:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 213419.4844iter count: 1.0 +[May 1, 2026 - 06:25:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 7.5638 iter count: 1.0 +[May 1, 2026 - 06:25:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 2150992.5000iter count: 1.0 +[May 1, 2026 - 06:25:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5235iter count: 1.0 +[May 1, 2026 - 06:25:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.1607 iter count: 1.0 +[May 1, 2026 - 06:25:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 556.3978iter count: 1.0 +[May 1, 2026 - 06:25:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.2457 iter count: 1.0 +[May 1, 2026 - 06:25:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 659.4407iter count: 1.0 +[May 1, 2026 - 06:25:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.1291 iter count: 1.0 +[May 1, 2026 - 06:25:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 522.2609iter count: 1.0 +[May 1, 2026 - 06:25:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.7383 iter count: 1.0 +[May 1, 2026 - 06:25:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 1766.3512iter count: 1.0 +[May 1, 2026 - 06:25:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 1, 2026 - 06:25:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:25:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 06:25:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452687 41.609776 41.323765 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.847656 67.855469 67.824219 +util 94.500000 100.000000 89.000000 +[May 1, 2026 - 06:25:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 411 : avg iteration time 50.77 seconds +[May 1, 2026 - 06:25:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 411 : data loading time 0.00 +[May 1, 2026 - 06:25:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 411 : avg forward pass time 14.43 +[May 1, 2026 - 06:25:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 411 : backward pass time 10.16 +[May 1, 2026 - 06:25:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 411 : optimizer step time 1.58 +[May 1, 2026 - 06:25:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 411-------------------- +[May 1, 2026 - 06:25:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0111 iter count: 1.0 +[May 1, 2026 - 06:25:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 1, 2026 - 06:25:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:25:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 06:26:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452932 41.610756 41.323765 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.789062 67.824219 67.777344 +util 95.250000 100.000000 90.000000 +[May 1, 2026 - 06:26:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 412 : avg iteration time 46.37 seconds +[May 1, 2026 - 06:26:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 412 : data loading time 0.00 +[May 1, 2026 - 06:26:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 412 : avg forward pass time 12.24 +[May 1, 2026 - 06:26:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 412 : backward pass time 10.14 +[May 1, 2026 - 06:26:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 412 : optimizer step time 1.58 +[May 1, 2026 - 06:26:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 412-------------------- +[May 1, 2026 - 06:26:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 1, 2026 - 06:26:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 1, 2026 - 06:26:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:26:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 06:27:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452974 41.610912 41.323765 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.784180 67.796875 67.746094 +util 97.250000 100.000000 93.000000 +[May 1, 2026 - 06:27:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 413 : avg iteration time 46.30 seconds +[May 1, 2026 - 06:27:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 413 : data loading time 0.00 +[May 1, 2026 - 06:27:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 413 : avg forward pass time 12.21 +[May 1, 2026 - 06:27:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 413 : backward pass time 10.15 +[May 1, 2026 - 06:27:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 413 : optimizer step time 1.58 +[May 1, 2026 - 06:27:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 413-------------------- +[May 1, 2026 - 06:27:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0100 iter count: 1.0 +[May 1, 2026 - 06:27:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 1, 2026 - 06:27:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:27:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 06:28:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452971 41.610889 41.323784 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.940430 67.992188 67.785156 +util 96.250000 100.000000 92.000000 +[May 1, 2026 - 06:28:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 414 : avg iteration time 46.30 seconds +[May 1, 2026 - 06:28:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 414 : data loading time 0.00 +[May 1, 2026 - 06:28:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 414 : avg forward pass time 12.22 +[May 1, 2026 - 06:28:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 414 : backward pass time 10.14 +[May 1, 2026 - 06:28:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 414 : optimizer step time 1.57 +[May 1, 2026 - 06:28:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 414-------------------- +[May 1, 2026 - 06:28:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 06:28:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 06:28:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:28:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 06:30:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452257 41.608009 41.323780 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.022949 136.031250 135.998047 +util 67.000000 74.000000 60.000000 +[May 1, 2026 - 06:30:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 415 : avg iteration time 163.20 seconds +[May 1, 2026 - 06:30:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 415 : data loading time 0.00 +[May 1, 2026 - 06:30:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 415 : avg forward pass time 72.98 +[May 1, 2026 - 06:30:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 415 : backward pass time 8.44 +[May 1, 2026 - 06:30:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 415 : optimizer step time 0.39 +[May 1, 2026 - 06:30:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 415-------------------- +[May 1, 2026 - 06:30:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 1462996.2704 iter count: 1.0 +[May 1, 2026 - 06:30:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5880 iter count: 1.0 +[May 1, 2026 - 06:30:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 1462996.2704iter count: 1.0 +[May 1, 2026 - 06:30:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:30:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.7474iter count: 1.0 +[May 1, 2026 - 06:30:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.3580 iter count: 1.0 +[May 1, 2026 - 06:30:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.9415 iter count: 1.0 +[May 1, 2026 - 06:30:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 1969637.1250iter count: 1.0 +[May 1, 2026 - 06:30:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 15.1193iter count: 1.0 +[May 1, 2026 - 06:30:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 58408988.0000iter count: 1.0 +[May 1, 2026 - 06:30:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5880iter count: 1.0 +[May 1, 2026 - 06:30:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.1615 iter count: 1.0 +[May 1, 2026 - 06:30:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 4117.1958iter count: 1.0 +[May 1, 2026 - 06:30:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.7448 iter count: 1.0 +[May 1, 2026 - 06:30:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 13220.1836iter count: 1.0 +[May 1, 2026 - 06:30:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.7200 iter count: 1.0 +[May 1, 2026 - 06:30:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 12580.8125iter count: 1.0 +[May 1, 2026 - 06:30:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 1.3580 iter count: 1.0 +[May 1, 2026 - 06:30:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 15.1193 iter count: 1.0 +[May 1, 2026 - 06:30:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 06:30:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:30:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 06:31:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452379 41.608486 41.323765 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.695801 67.796875 67.392578 +util 95.500000 100.000000 91.000000 +[May 1, 2026 - 06:31:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 416 : avg iteration time 47.78 seconds +[May 1, 2026 - 06:31:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 416 : data loading time 0.00 +[May 1, 2026 - 06:31:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 416 : avg forward pass time 12.98 +[May 1, 2026 - 06:31:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 416 : backward pass time 10.13 +[May 1, 2026 - 06:31:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 416 : optimizer step time 1.56 +[May 1, 2026 - 06:31:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 416-------------------- +[May 1, 2026 - 06:31:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0118 iter count: 1.0 +[May 1, 2026 - 06:31:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0118 iter count: 1.0 +[May 1, 2026 - 06:31:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:31:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 06:32:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452549 41.609169 41.323772 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.817871 67.953125 67.412109 +util 98.000000 100.000000 95.000000 +[May 1, 2026 - 06:32:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 417 : avg iteration time 43.53 seconds +[May 1, 2026 - 06:32:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 417 : data loading time 0.00 +[May 1, 2026 - 06:32:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 417 : avg forward pass time 10.84 +[May 1, 2026 - 06:32:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 417 : backward pass time 10.12 +[May 1, 2026 - 06:32:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 417 : optimizer step time 1.59 +[May 1, 2026 - 06:32:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 417-------------------- +[May 1, 2026 - 06:32:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 06:32:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 06:32:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:32:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 06:33:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452632 41.609459 41.323772 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.803223 67.933594 67.412109 +util 94.500000 99.000000 90.000000 +[May 1, 2026 - 06:33:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 418 : avg iteration time 48.93 seconds +[May 1, 2026 - 06:33:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 418 : data loading time 0.00 +[May 1, 2026 - 06:33:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 418 : avg forward pass time 13.54 +[May 1, 2026 - 06:33:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 418 : backward pass time 10.15 +[May 1, 2026 - 06:33:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 418 : optimizer step time 1.56 +[May 1, 2026 - 06:33:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 418-------------------- +[May 1, 2026 - 06:33:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 06:33:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 06:33:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:33:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 06:34:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452673 41.609650 41.323765 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.466309 67.484375 67.412109 +util 94.250000 99.000000 89.000000 +[May 1, 2026 - 06:34:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 419 : avg iteration time 48.96 seconds +[May 1, 2026 - 06:34:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 419 : data loading time 0.00 +[May 1, 2026 - 06:34:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 419 : avg forward pass time 13.55 +[May 1, 2026 - 06:34:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 419 : backward pass time 10.15 +[May 1, 2026 - 06:34:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 419 : optimizer step time 1.56 +[May 1, 2026 - 06:34:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 419-------------------- +[May 1, 2026 - 06:34:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 1, 2026 - 06:34:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 1, 2026 - 06:34:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:34:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 06:36:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452353 41.608372 41.323765 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.065430 136.068359 136.056641 +util 63.500000 69.000000 56.000000 +[May 1, 2026 - 06:36:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 420 : avg iteration time 168.73 seconds +[May 1, 2026 - 06:36:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 420 : data loading time 0.00 +[May 1, 2026 - 06:36:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 420 : avg forward pass time 75.73 +[May 1, 2026 - 06:36:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 420 : backward pass time 8.44 +[May 1, 2026 - 06:36:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 420 : optimizer step time 0.40 +[May 1, 2026 - 06:36:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 420-------------------- +[May 1, 2026 - 06:36:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 322653.5840 iter count: 1.0 +[May 1, 2026 - 06:36:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6041 iter count: 1.0 +[May 1, 2026 - 06:36:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 322653.5840iter count: 1.0 +[May 1, 2026 - 06:36:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:36:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 5.3024iter count: 1.0 +[May 1, 2026 - 06:36:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 2.2783 iter count: 1.0 +[May 1, 2026 - 06:36:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.8627 iter count: 1.0 +[May 1, 2026 - 06:36:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 552301.6875iter count: 1.0 +[May 1, 2026 - 06:36:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 95.2537iter count: 1.0 +[May 1, 2026 - 06:36:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 6752187.5000iter count: 1.0 +[May 1, 2026 - 06:36:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6041iter count: 1.0 +[May 1, 2026 - 06:36:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 6.7507 iter count: 1.0 +[May 1, 2026 - 06:36:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 730477.2500iter count: 1.0 +[May 1, 2026 - 06:36:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 6.6532 iter count: 1.0 +[May 1, 2026 - 06:36:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 601031.2500iter count: 1.0 +[May 1, 2026 - 06:36:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.2347 iter count: 1.0 +[May 1, 2026 - 06:36:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 35217.7031iter count: 1.0 +[May 1, 2026 - 06:36:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.0171 iter count: 1.0 +[May 1, 2026 - 06:36:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 168417.5625iter count: 1.0 +[May 1, 2026 - 06:36:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 1, 2026 - 06:36:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:36:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 06:37:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452408 41.608589 41.323765 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.949707 68.089844 67.529297 +util 95.750000 100.000000 91.000000 +[May 1, 2026 - 06:37:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 421 : avg iteration time 50.64 seconds +[May 1, 2026 - 06:37:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 421 : data loading time 0.00 +[May 1, 2026 - 06:37:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 421 : avg forward pass time 14.39 +[May 1, 2026 - 06:37:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 421 : backward pass time 10.14 +[May 1, 2026 - 06:37:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 421 : optimizer step time 1.56 +[May 1, 2026 - 06:37:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 421-------------------- +[May 1, 2026 - 06:37:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 06:37:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 06:37:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:37:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 06:38:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452653 41.609570 41.323765 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.827637 67.933594 67.509766 +util 95.500000 99.000000 91.000000 +[May 1, 2026 - 06:38:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 422 : avg iteration time 43.59 seconds +[May 1, 2026 - 06:38:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 422 : data loading time 0.00 +[May 1, 2026 - 06:38:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 422 : avg forward pass time 10.86 +[May 1, 2026 - 06:38:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 422 : backward pass time 10.15 +[May 1, 2026 - 06:38:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 422 : optimizer step time 1.57 +[May 1, 2026 - 06:38:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 422-------------------- +[May 1, 2026 - 06:38:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0114 iter count: 1.0 +[May 1, 2026 - 06:38:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 1, 2026 - 06:38:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:38:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 06:39:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452774 41.610054 41.323765 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.905762 68.031250 67.529297 +util 95.250000 100.000000 90.000000 +[May 1, 2026 - 06:39:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 423 : avg iteration time 46.39 seconds +[May 1, 2026 - 06:39:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 423 : data loading time 0.00 +[May 1, 2026 - 06:39:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 423 : avg forward pass time 12.26 +[May 1, 2026 - 06:39:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 423 : backward pass time 10.15 +[May 1, 2026 - 06:39:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 423 : optimizer step time 1.58 +[May 1, 2026 - 06:39:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 423-------------------- +[May 1, 2026 - 06:39:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0111 iter count: 1.0 +[May 1, 2026 - 06:39:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 1, 2026 - 06:39:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:39:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 06:40:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452772 41.610046 41.323765 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.910645 68.031250 67.548828 +util 94.250000 100.000000 89.000000 +[May 1, 2026 - 06:40:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 424 : avg iteration time 46.38 seconds +[May 1, 2026 - 06:40:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 424 : data loading time 0.00 +[May 1, 2026 - 06:40:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 424 : avg forward pass time 12.24 +[May 1, 2026 - 06:40:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 424 : backward pass time 10.16 +[May 1, 2026 - 06:40:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 424 : optimizer step time 1.57 +[May 1, 2026 - 06:40:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 424-------------------- +[May 1, 2026 - 06:40:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 06:40:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 06:40:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:40:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 06:42:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452315 41.608189 41.323784 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 135.988770 136.037109 135.972656 +util 62.500000 67.000000 58.000000 +[May 1, 2026 - 06:42:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 425 : avg iteration time 160.49 seconds +[May 1, 2026 - 06:42:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 425 : data loading time 0.00 +[May 1, 2026 - 06:42:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 425 : avg forward pass time 71.65 +[May 1, 2026 - 06:42:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 425 : backward pass time 8.41 +[May 1, 2026 - 06:42:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 425 : optimizer step time 0.39 +[May 1, 2026 - 06:42:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 425-------------------- +[May 1, 2026 - 06:42:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 28455.4076 iter count: 1.0 +[May 1, 2026 - 06:42:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5121 iter count: 1.0 +[May 1, 2026 - 06:42:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 28455.4076iter count: 1.0 +[May 1, 2026 - 06:42:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:42:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 3.9764iter count: 1.0 +[May 1, 2026 - 06:42:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.3913 iter count: 1.0 +[May 1, 2026 - 06:42:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 6.9178 iter count: 1.0 +[May 1, 2026 - 06:42:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 58359.9336iter count: 1.0 +[May 1, 2026 - 06:42:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 2.1872 iter count: 1.0 +[May 1, 2026 - 06:42:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 1020276.9375iter count: 1.0 +[May 1, 2026 - 06:42:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5121iter count: 1.0 +[May 1, 2026 - 06:42:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.4051 iter count: 1.0 +[May 1, 2026 - 06:42:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 6701.6826iter count: 1.0 +[May 1, 2026 - 06:42:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.1400 iter count: 1.0 +[May 1, 2026 - 06:42:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 3943.9790iter count: 1.0 +[May 1, 2026 - 06:42:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 2.4251 iter count: 1.0 +[May 1, 2026 - 06:42:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 127.7629iter count: 1.0 +[May 1, 2026 - 06:42:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.9178 iter count: 1.0 +[May 1, 2026 - 06:42:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 1020276.9375iter count: 1.0 +[May 1, 2026 - 06:42:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 06:42:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:42:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 06:43:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452313 41.608189 41.323776 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.077148 68.343750 67.277344 +util 96.000000 100.000000 91.000000 +[May 1, 2026 - 06:43:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 426 : avg iteration time 50.42 seconds +[May 1, 2026 - 06:43:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 426 : data loading time 0.00 +[May 1, 2026 - 06:43:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 426 : avg forward pass time 14.29 +[May 1, 2026 - 06:43:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 426 : backward pass time 10.15 +[May 1, 2026 - 06:43:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 426 : optimizer step time 1.56 +[May 1, 2026 - 06:43:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 426-------------------- +[May 1, 2026 - 06:43:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0113 iter count: 1.0 +[May 1, 2026 - 06:43:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 1, 2026 - 06:43:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:43:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 06:44:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452558 41.609158 41.323776 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.125977 68.343750 67.472656 +util 95.750000 100.000000 91.000000 +[May 1, 2026 - 06:44:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 427 : avg iteration time 48.91 seconds +[May 1, 2026 - 06:44:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 427 : data loading time 0.00 +[May 1, 2026 - 06:44:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 427 : avg forward pass time 13.54 +[May 1, 2026 - 06:44:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 427 : backward pass time 10.15 +[May 1, 2026 - 06:44:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 427 : optimizer step time 1.56 +[May 1, 2026 - 06:44:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 427-------------------- +[May 1, 2026 - 06:44:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 06:44:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 06:44:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:44:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 06:45:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452680 41.609646 41.323776 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.125977 68.382812 67.355469 +util 96.750000 99.000000 93.000000 +[May 1, 2026 - 06:45:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 428 : avg iteration time 46.31 seconds +[May 1, 2026 - 06:45:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 428 : data loading time 0.00 +[May 1, 2026 - 06:45:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 428 : avg forward pass time 12.22 +[May 1, 2026 - 06:45:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 428 : backward pass time 10.16 +[May 1, 2026 - 06:45:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 428 : optimizer step time 1.57 +[May 1, 2026 - 06:45:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 428-------------------- +[May 1, 2026 - 06:45:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0113 iter count: 1.0 +[May 1, 2026 - 06:45:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 1, 2026 - 06:45:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:45:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 06:45:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452747 41.609894 41.323776 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.140625 68.343750 67.531250 +util 98.250000 99.000000 96.000000 +[May 1, 2026 - 06:45:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 429 : avg iteration time 46.23 seconds +[May 1, 2026 - 06:45:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 429 : data loading time 0.00 +[May 1, 2026 - 06:45:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 429 : avg forward pass time 12.20 +[May 1, 2026 - 06:45:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 429 : backward pass time 10.13 +[May 1, 2026 - 06:45:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 429 : optimizer step time 1.56 +[May 1, 2026 - 06:45:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 429-------------------- +[May 1, 2026 - 06:45:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 06:45:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 06:45:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:45:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 06:48:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452299 41.608112 41.323776 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.031250 136.037109 136.029297 +util 66.000000 72.000000 60.000000 +[May 1, 2026 - 06:48:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 430 : avg iteration time 168.86 seconds +[May 1, 2026 - 06:48:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 430 : data loading time 0.00 +[May 1, 2026 - 06:48:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 430 : avg forward pass time 75.81 +[May 1, 2026 - 06:48:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 430 : backward pass time 8.46 +[May 1, 2026 - 06:48:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 430 : optimizer step time 0.40 +[May 1, 2026 - 06:48:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 430-------------------- +[May 1, 2026 - 06:48:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 1019681.5059 iter count: 1.0 +[May 1, 2026 - 06:48:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5862 iter count: 1.0 +[May 1, 2026 - 06:48:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 1019681.5059iter count: 1.0 +[May 1, 2026 - 06:48:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:48:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 5.1748iter count: 1.0 +[May 1, 2026 - 06:48:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.7247 iter count: 1.0 +[May 1, 2026 - 06:48:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.6556 iter count: 1.0 +[May 1, 2026 - 06:48:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 1523443.2500iter count: 1.0 +[May 1, 2026 - 06:48:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 31.4793iter count: 1.0 +[May 1, 2026 - 06:48:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 32971350.0000iter count: 1.0 +[May 1, 2026 - 06:48:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5862iter count: 1.0 +[May 1, 2026 - 06:48:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.9091 iter count: 1.0 +[May 1, 2026 - 06:48:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 2485.4541iter count: 1.0 +[May 1, 2026 - 06:48:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.0027 iter count: 1.0 +[May 1, 2026 - 06:48:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 2997.2236iter count: 1.0 +[May 1, 2026 - 06:48:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 6.6164 iter count: 1.0 +[May 1, 2026 - 06:48:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 558384.6875iter count: 1.0 +[May 1, 2026 - 06:48:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 2.3780 iter count: 1.0 +[May 1, 2026 - 06:48:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 116.2753iter count: 1.0 +[May 1, 2026 - 06:48:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 1, 2026 - 06:48:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:48:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 06:49:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452297 41.608116 41.323776 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.813477 67.953125 67.394531 +util 96.500000 100.000000 92.000000 +[May 1, 2026 - 06:49:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 431 : avg iteration time 47.90 seconds +[May 1, 2026 - 06:49:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 431 : data loading time 0.00 +[May 1, 2026 - 06:49:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 431 : avg forward pass time 13.02 +[May 1, 2026 - 06:49:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 431 : backward pass time 10.14 +[May 1, 2026 - 06:49:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 431 : optimizer step time 1.58 +[May 1, 2026 - 06:49:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 431-------------------- +[May 1, 2026 - 06:49:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 1, 2026 - 06:49:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 1, 2026 - 06:49:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:49:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 06:50:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452731 41.609840 41.323788 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.823242 67.972656 67.375000 +util 93.750000 100.000000 88.000000 +[May 1, 2026 - 06:50:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 432 : avg iteration time 46.31 seconds +[May 1, 2026 - 06:50:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 432 : data loading time 0.00 +[May 1, 2026 - 06:50:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 432 : avg forward pass time 12.23 +[May 1, 2026 - 06:50:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 432 : backward pass time 10.13 +[May 1, 2026 - 06:50:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 432 : optimizer step time 1.56 +[May 1, 2026 - 06:50:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 432-------------------- +[May 1, 2026 - 06:50:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0112 iter count: 1.0 +[May 1, 2026 - 06:50:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 1, 2026 - 06:50:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:50:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 06:51:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452914 41.610584 41.323776 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.583984 67.601562 67.531250 +util 93.500000 99.000000 88.000000 +[May 1, 2026 - 06:51:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 433 : avg iteration time 43.60 seconds +[May 1, 2026 - 06:51:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 433 : data loading time 0.00 +[May 1, 2026 - 06:51:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 433 : avg forward pass time 10.86 +[May 1, 2026 - 06:51:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 433 : backward pass time 10.14 +[May 1, 2026 - 06:51:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 433 : optimizer step time 1.59 +[May 1, 2026 - 06:51:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 433-------------------- +[May 1, 2026 - 06:51:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 06:51:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 06:51:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:51:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 06:51:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452927 41.610611 41.323799 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.759766 67.835938 67.531250 +util 98.750000 100.000000 96.000000 +[May 1, 2026 - 06:51:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 434 : avg iteration time 46.32 seconds +[May 1, 2026 - 06:51:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 434 : data loading time 0.00 +[May 1, 2026 - 06:51:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 434 : avg forward pass time 12.23 +[May 1, 2026 - 06:51:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 434 : backward pass time 10.14 +[May 1, 2026 - 06:51:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 434 : optimizer step time 1.57 +[May 1, 2026 - 06:51:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 434-------------------- +[May 1, 2026 - 06:51:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 1, 2026 - 06:51:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 1, 2026 - 06:51:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:51:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 06:54:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452412 41.608574 41.323776 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.003418 136.011719 135.978516 +util 65.500000 70.000000 61.000000 +[May 1, 2026 - 06:54:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 435 : avg iteration time 165.75 seconds +[May 1, 2026 - 06:54:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 435 : data loading time 0.00 +[May 1, 2026 - 06:54:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 435 : avg forward pass time 74.24 +[May 1, 2026 - 06:54:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 435 : backward pass time 8.44 +[May 1, 2026 - 06:54:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 435 : optimizer step time 0.39 +[May 1, 2026 - 06:54:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 435-------------------- +[May 1, 2026 - 06:54:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 1123430.6016 iter count: 1.0 +[May 1, 2026 - 06:54:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5539 iter count: 1.0 +[May 1, 2026 - 06:54:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 1123430.6016iter count: 1.0 +[May 1, 2026 - 06:54:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:54:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.8655iter count: 1.0 +[May 1, 2026 - 06:54:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.7348 iter count: 1.0 +[May 1, 2026 - 06:54:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.6313 iter count: 1.0 +[May 1, 2026 - 06:54:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 1729483.2500iter count: 1.0 +[May 1, 2026 - 06:54:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 32.1248iter count: 1.0 +[May 1, 2026 - 06:54:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 31406264.0000iter count: 1.0 +[May 1, 2026 - 06:54:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5539iter count: 1.0 +[May 1, 2026 - 06:54:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.0791 iter count: 1.0 +[May 1, 2026 - 06:54:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 25801.4004iter count: 1.0 +[May 1, 2026 - 06:54:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 1.7348 iter count: 1.0 +[May 1, 2026 - 06:54:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 32.1248 iter count: 1.0 +[May 1, 2026 - 06:54:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.8902 iter count: 1.0 +[May 1, 2026 - 06:54:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 130675.1719iter count: 1.0 +[May 1, 2026 - 06:54:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 2.3173 iter count: 1.0 +[May 1, 2026 - 06:54:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 102.9900iter count: 1.0 +[May 1, 2026 - 06:54:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 1, 2026 - 06:54:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:54:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 06:55:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452418 41.608562 41.323788 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.876953 67.992188 67.531250 +util 98.000000 100.000000 95.000000 +[May 1, 2026 - 06:55:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 436 : avg iteration time 47.84 seconds +[May 1, 2026 - 06:55:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 436 : data loading time 0.00 +[May 1, 2026 - 06:55:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 436 : avg forward pass time 12.99 +[May 1, 2026 - 06:55:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 436 : backward pass time 10.16 +[May 1, 2026 - 06:55:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 436 : optimizer step time 1.57 +[May 1, 2026 - 06:55:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 436-------------------- +[May 1, 2026 - 06:55:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0132 iter count: 1.0 +[May 1, 2026 - 06:55:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0132 iter count: 1.0 +[May 1, 2026 - 06:55:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:55:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 06:56:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452785 41.610008 41.323788 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.876953 68.011719 67.472656 +util 95.750000 99.000000 91.000000 +[May 1, 2026 - 06:56:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 437 : avg iteration time 48.96 seconds +[May 1, 2026 - 06:56:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 437 : data loading time 0.00 +[May 1, 2026 - 06:56:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 437 : avg forward pass time 13.55 +[May 1, 2026 - 06:56:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 437 : backward pass time 10.14 +[May 1, 2026 - 06:56:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 437 : optimizer step time 1.58 +[May 1, 2026 - 06:56:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 437-------------------- +[May 1, 2026 - 06:56:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0104 iter count: 1.0 +[May 1, 2026 - 06:56:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 1, 2026 - 06:56:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:56:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 06:56:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452782 41.610004 41.323788 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.916016 68.050781 67.511719 +util 95.250000 99.000000 91.000000 +[May 1, 2026 - 06:56:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 438 : avg iteration time 43.50 seconds +[May 1, 2026 - 06:56:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 438 : data loading time 0.00 +[May 1, 2026 - 06:56:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 438 : avg forward pass time 10.83 +[May 1, 2026 - 06:56:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 438 : backward pass time 10.15 +[May 1, 2026 - 06:56:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 438 : optimizer step time 1.55 +[May 1, 2026 - 06:56:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 438-------------------- +[May 1, 2026 - 06:56:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0113 iter count: 1.0 +[May 1, 2026 - 06:56:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 1, 2026 - 06:56:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:56:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 06:57:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452782 41.610004 41.323799 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.872070 68.011719 67.453125 +util 93.000000 97.000000 88.000000 +[May 1, 2026 - 06:57:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 439 : avg iteration time 46.25 seconds +[May 1, 2026 - 06:57:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 439 : data loading time 0.00 +[May 1, 2026 - 06:57:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 439 : avg forward pass time 12.21 +[May 1, 2026 - 06:57:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 439 : backward pass time 10.14 +[May 1, 2026 - 06:57:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 439 : optimizer step time 1.56 +[May 1, 2026 - 06:57:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 439-------------------- +[May 1, 2026 - 06:57:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 1, 2026 - 06:57:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 1, 2026 - 06:57:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 06:57:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 07:00:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452458 41.608723 41.323788 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.027832 136.076172 136.011719 +util 63.000000 70.000000 57.000000 +[May 1, 2026 - 07:00:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 440 : avg iteration time 171.12 seconds +[May 1, 2026 - 07:00:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 440 : data loading time 0.00 +[May 1, 2026 - 07:00:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 440 : avg forward pass time 76.97 +[May 1, 2026 - 07:00:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 440 : backward pass time 8.40 +[May 1, 2026 - 07:00:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 440 : optimizer step time 0.39 +[May 1, 2026 - 07:00:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 440-------------------- +[May 1, 2026 - 07:00:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 258384.7113 iter count: 1.0 +[May 1, 2026 - 07:00:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5922 iter count: 1.0 +[May 1, 2026 - 07:00:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 258384.7113iter count: 1.0 +[May 1, 2026 - 07:00:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:00:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 3.7910iter count: 1.0 +[May 1, 2026 - 07:00:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.7885 iter count: 1.0 +[May 1, 2026 - 07:00:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.0816 iter count: 1.0 +[May 1, 2026 - 07:00:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 354375.1875iter count: 1.0 +[May 1, 2026 - 07:00:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 4.8405 iter count: 1.0 +[May 1, 2026 - 07:00:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 10460416.0000iter count: 1.0 +[May 1, 2026 - 07:00:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5922iter count: 1.0 +[May 1, 2026 - 07:00:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.2869 iter count: 1.0 +[May 1, 2026 - 07:00:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 39098.0430iter count: 1.0 +[May 1, 2026 - 07:00:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 2.6588 iter count: 1.0 +[May 1, 2026 - 07:00:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 203.8851iter count: 1.0 +[May 1, 2026 - 07:00:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.5624 iter count: 1.0 +[May 1, 2026 - 07:00:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 1242.4918iter count: 1.0 +[May 1, 2026 - 07:00:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.9931 iter count: 1.0 +[May 1, 2026 - 07:00:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 21726.2734iter count: 1.0 +[May 1, 2026 - 07:00:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 07:00:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:00:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 07:01:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452526 41.608971 41.323788 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.881836 67.992188 67.550781 +util 95.000000 99.000000 90.000000 +[May 1, 2026 - 07:01:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 441 : avg iteration time 45.11 seconds +[May 1, 2026 - 07:01:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 441 : data loading time 0.00 +[May 1, 2026 - 07:01:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 441 : avg forward pass time 11.63 +[May 1, 2026 - 07:01:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 441 : backward pass time 10.14 +[May 1, 2026 - 07:01:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 441 : optimizer step time 1.57 +[May 1, 2026 - 07:01:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 441-------------------- +[May 1, 2026 - 07:01:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0120 iter count: 1.0 +[May 1, 2026 - 07:01:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0120 iter count: 1.0 +[May 1, 2026 - 07:01:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:01:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 07:02:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452645 41.609432 41.323811 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.867188 67.992188 67.492188 +util 95.500000 100.000000 91.000000 +[May 1, 2026 - 07:02:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 442 : avg iteration time 46.31 seconds +[May 1, 2026 - 07:02:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 442 : data loading time 0.00 +[May 1, 2026 - 07:02:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 442 : avg forward pass time 12.22 +[May 1, 2026 - 07:02:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 442 : backward pass time 10.15 +[May 1, 2026 - 07:02:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 442 : optimizer step time 1.58 +[May 1, 2026 - 07:02:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 442-------------------- +[May 1, 2026 - 07:02:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 07:02:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 07:02:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:02:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 07:02:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452646 41.609432 41.323788 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.720703 67.796875 67.492188 +util 92.250000 99.000000 86.000000 +[May 1, 2026 - 07:02:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 443 : avg iteration time 43.56 seconds +[May 1, 2026 - 07:02:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 443 : data loading time 0.00 +[May 1, 2026 - 07:02:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 443 : avg forward pass time 10.86 +[May 1, 2026 - 07:02:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 443 : backward pass time 10.14 +[May 1, 2026 - 07:02:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 443 : optimizer step time 1.56 +[May 1, 2026 - 07:02:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 443-------------------- +[May 1, 2026 - 07:02:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 1, 2026 - 07:02:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 1, 2026 - 07:02:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:02:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 07:03:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452636 41.609432 41.323788 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.867188 67.992188 67.492188 +util 98.000000 100.000000 95.000000 +[May 1, 2026 - 07:03:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 444 : avg iteration time 46.19 seconds +[May 1, 2026 - 07:03:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 444 : data loading time 0.00 +[May 1, 2026 - 07:03:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 444 : avg forward pass time 12.18 +[May 1, 2026 - 07:03:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 444 : backward pass time 10.14 +[May 1, 2026 - 07:03:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 444 : optimizer step time 1.56 +[May 1, 2026 - 07:03:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 444-------------------- +[May 1, 2026 - 07:03:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 07:03:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 07:03:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:03:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 07:06:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452463 41.608738 41.323792 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.027832 136.076172 136.011719 +util 66.000000 73.000000 60.000000 +[May 1, 2026 - 07:06:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 445 : avg iteration time 168.67 seconds +[May 1, 2026 - 07:06:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 445 : data loading time 0.00 +[May 1, 2026 - 07:06:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 445 : avg forward pass time 75.73 +[May 1, 2026 - 07:06:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 445 : backward pass time 8.43 +[May 1, 2026 - 07:06:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 445 : optimizer step time 0.39 +[May 1, 2026 - 07:06:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 445-------------------- +[May 1, 2026 - 07:06:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 506690.6118 iter count: 1.0 +[May 1, 2026 - 07:06:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5876 iter count: 1.0 +[May 1, 2026 - 07:06:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 506690.6118iter count: 1.0 +[May 1, 2026 - 07:06:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:06:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.5898iter count: 1.0 +[May 1, 2026 - 07:06:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.8346 iter count: 1.0 +[May 1, 2026 - 07:06:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.2358 iter count: 1.0 +[May 1, 2026 - 07:06:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 887867.1875iter count: 1.0 +[May 1, 2026 - 07:06:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 5.3082 iter count: 1.0 +[May 1, 2026 - 07:06:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 14239915.0000iter count: 1.0 +[May 1, 2026 - 07:06:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5876iter count: 1.0 +[May 1, 2026 - 07:06:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.8946 iter count: 1.0 +[May 1, 2026 - 07:06:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 2414.5830iter count: 1.0 +[May 1, 2026 - 07:06:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.5826 iter count: 1.0 +[May 1, 2026 - 07:06:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 70630.1016iter count: 1.0 +[May 1, 2026 - 07:06:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.4059 iter count: 1.0 +[May 1, 2026 - 07:06:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 908.5168iter count: 1.0 +[May 1, 2026 - 07:06:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.9391 iter count: 1.0 +[May 1, 2026 - 07:06:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 144085.5312iter count: 1.0 +[May 1, 2026 - 07:06:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 1, 2026 - 07:06:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:06:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 07:07:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452479 41.608761 41.323788 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.818359 67.953125 67.414062 +util 92.500000 99.000000 87.000000 +[May 1, 2026 - 07:07:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 446 : avg iteration time 50.54 seconds +[May 1, 2026 - 07:07:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 446 : data loading time 0.00 +[May 1, 2026 - 07:07:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 446 : avg forward pass time 14.35 +[May 1, 2026 - 07:07:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 446 : backward pass time 10.14 +[May 1, 2026 - 07:07:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 446 : optimizer step time 1.57 +[May 1, 2026 - 07:07:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 446-------------------- +[May 1, 2026 - 07:07:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0104 iter count: 1.0 +[May 1, 2026 - 07:07:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 1, 2026 - 07:07:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:07:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 07:07:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452910 41.610481 41.323788 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.666992 67.757812 67.394531 +util 95.500000 100.000000 90.000000 +[May 1, 2026 - 07:07:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 447 : avg iteration time 46.33 seconds +[May 1, 2026 - 07:07:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 447 : data loading time 0.00 +[May 1, 2026 - 07:07:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 447 : avg forward pass time 12.25 +[May 1, 2026 - 07:07:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 447 : backward pass time 10.14 +[May 1, 2026 - 07:07:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 447 : optimizer step time 1.56 +[May 1, 2026 - 07:07:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 447-------------------- +[May 1, 2026 - 07:07:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0111 iter count: 1.0 +[May 1, 2026 - 07:07:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 1, 2026 - 07:07:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:07:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 07:08:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.453021 41.610939 41.323803 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.544922 67.562500 67.492188 +util 96.750000 99.000000 93.000000 +[May 1, 2026 - 07:08:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 448 : avg iteration time 49.03 seconds +[May 1, 2026 - 07:08:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 448 : data loading time 0.00 +[May 1, 2026 - 07:08:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 448 : avg forward pass time 13.57 +[May 1, 2026 - 07:08:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 448 : backward pass time 10.15 +[May 1, 2026 - 07:08:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 448 : optimizer step time 1.57 +[May 1, 2026 - 07:08:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 448-------------------- +[May 1, 2026 - 07:08:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0108 iter count: 1.0 +[May 1, 2026 - 07:08:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 1, 2026 - 07:08:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:08:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 07:09:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.453016 41.610947 41.323788 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.710938 67.777344 67.511719 +util 93.250000 99.000000 89.000000 +[May 1, 2026 - 07:09:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 449 : avg iteration time 46.30 seconds +[May 1, 2026 - 07:09:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 449 : data loading time 0.00 +[May 1, 2026 - 07:09:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 449 : avg forward pass time 12.22 +[May 1, 2026 - 07:09:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 449 : backward pass time 10.15 +[May 1, 2026 - 07:09:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 449 : optimizer step time 1.58 +[May 1, 2026 - 07:09:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 449-------------------- +[May 1, 2026 - 07:09:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 1, 2026 - 07:09:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 1, 2026 - 07:09:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:09:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 07:12:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452423 41.608574 41.323788 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.022949 136.031250 135.998047 +util 65.000000 69.000000 61.000000 +[May 1, 2026 - 07:12:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 450 : avg iteration time 166.09 seconds +[May 1, 2026 - 07:12:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 450 : data loading time 0.00 +[May 1, 2026 - 07:12:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 450 : avg forward pass time 74.42 +[May 1, 2026 - 07:12:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 450 : backward pass time 8.43 +[May 1, 2026 - 07:12:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 450 : optimizer step time 0.40 +[May 1, 2026 - 07:12:21 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 450-------------------- +[May 1, 2026 - 07:12:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 136943.6221 iter count: 1.0 +[May 1, 2026 - 07:12:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5980 iter count: 1.0 +[May 1, 2026 - 07:12:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 136943.6221iter count: 1.0 +[May 1, 2026 - 07:12:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:12:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.6555iter count: 1.0 +[May 1, 2026 - 07:12:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.7807 iter count: 1.0 +[May 1, 2026 - 07:12:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.4669 iter count: 1.0 +[May 1, 2026 - 07:12:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 222301.8906iter count: 1.0 +[May 1, 2026 - 07:12:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 4.7656 iter count: 1.0 +[May 1, 2026 - 07:12:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 3059790.5000iter count: 1.0 +[May 1, 2026 - 07:12:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5980iter count: 1.0 +[May 1, 2026 - 07:12:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.6530 iter count: 1.0 +[May 1, 2026 - 07:12:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 1489.2592iter count: 1.0 +[May 1, 2026 - 07:12:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.2047 iter count: 1.0 +[May 1, 2026 - 07:12:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 607.4798iter count: 1.0 +[May 1, 2026 - 07:12:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.1764 iter count: 1.0 +[May 1, 2026 - 07:12:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 31347.7168iter count: 1.0 +[May 1, 2026 - 07:12:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.7013 iter count: 1.0 +[May 1, 2026 - 07:12:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 12120.3984iter count: 1.0 +[May 1, 2026 - 07:12:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[May 1, 2026 - 07:12:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:12:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 07:13:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452575 41.609055 41.323811 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.632812 67.757812 67.257812 +util 95.250000 100.000000 90.000000 +[May 1, 2026 - 07:13:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 451 : avg iteration time 50.49 seconds +[May 1, 2026 - 07:13:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 451 : data loading time 0.00 +[May 1, 2026 - 07:13:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 451 : avg forward pass time 14.32 +[May 1, 2026 - 07:13:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 451 : backward pass time 10.14 +[May 1, 2026 - 07:13:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 451 : optimizer step time 1.57 +[May 1, 2026 - 07:13:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 451-------------------- +[May 1, 2026 - 07:13:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0124 iter count: 1.0 +[May 1, 2026 - 07:13:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0124 iter count: 1.0 +[May 1, 2026 - 07:13:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:13:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 07:13:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.452951 41.610561 41.323811 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.769531 67.933594 67.277344 +util 94.250000 100.000000 89.000000 +[May 1, 2026 - 07:13:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 452 : avg iteration time 46.24 seconds +[May 1, 2026 - 07:13:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 452 : data loading time 0.00 +[May 1, 2026 - 07:13:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 452 : avg forward pass time 12.20 +[May 1, 2026 - 07:13:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 452 : backward pass time 10.13 +[May 1, 2026 - 07:13:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 452 : optimizer step time 1.57 +[May 1, 2026 - 07:13:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 452-------------------- +[May 1, 2026 - 07:13:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0118 iter count: 1.0 +[May 1, 2026 - 07:13:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0118 iter count: 1.0 +[May 1, 2026 - 07:13:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:13:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 07:14:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.453077 41.611065 41.323811 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.818359 67.933594 67.472656 +util 93.250000 99.000000 88.000000 +[May 1, 2026 - 07:14:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 453 : avg iteration time 48.93 seconds +[May 1, 2026 - 07:14:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 453 : data loading time 0.00 +[May 1, 2026 - 07:14:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 453 : avg forward pass time 13.52 +[May 1, 2026 - 07:14:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 453 : backward pass time 10.15 +[May 1, 2026 - 07:14:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 453 : optimizer step time 1.57 +[May 1, 2026 - 07:14:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 453-------------------- +[May 1, 2026 - 07:14:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0125 iter count: 1.0 +[May 1, 2026 - 07:14:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0125 iter count: 1.0 +[May 1, 2026 - 07:14:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:14:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 07:15:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.659870 41.818546 41.527100 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.608398 67.718750 67.277344 +util 96.250000 100.000000 91.000000 +[May 1, 2026 - 07:15:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 454 : avg iteration time 48.93 seconds +[May 1, 2026 - 07:15:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 454 : data loading time 0.00 +[May 1, 2026 - 07:15:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 454 : avg forward pass time 13.00 +[May 1, 2026 - 07:15:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 454 : backward pass time 10.13 +[May 1, 2026 - 07:15:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 454 : optimizer step time 1.57 +[May 1, 2026 - 07:15:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 454-------------------- +[May 1, 2026 - 07:15:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 07:15:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 07:15:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:15:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 07:18:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.659855 41.816475 41.527893 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.016602 136.037109 136.009766 +util 61.750000 68.000000 56.000000 +[May 1, 2026 - 07:18:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 455 : avg iteration time 163.27 seconds +[May 1, 2026 - 07:18:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 455 : data loading time 0.00 +[May 1, 2026 - 07:18:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 455 : avg forward pass time 73.04 +[May 1, 2026 - 07:18:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 455 : backward pass time 8.44 +[May 1, 2026 - 07:18:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 455 : optimizer step time 0.40 +[May 1, 2026 - 07:18:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 455-------------------- +[May 1, 2026 - 07:18:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 358549.6377 iter count: 1.0 +[May 1, 2026 - 07:18:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5212 iter count: 1.0 +[May 1, 2026 - 07:18:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 358549.6377iter count: 1.0 +[May 1, 2026 - 07:18:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:18:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.8584iter count: 1.0 +[May 1, 2026 - 07:18:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.9441 iter count: 1.0 +[May 1, 2026 - 07:18:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.9913 iter count: 1.0 +[May 1, 2026 - 07:18:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 712265.3750iter count: 1.0 +[May 1, 2026 - 07:18:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 6.6073 iter count: 1.0 +[May 1, 2026 - 07:18:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 8733198.0000iter count: 1.0 +[May 1, 2026 - 07:18:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5212iter count: 1.0 +[May 1, 2026 - 07:18:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 6.0790 iter count: 1.0 +[May 1, 2026 - 07:18:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 190615.8125iter count: 1.0 +[May 1, 2026 - 07:18:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.2655 iter count: 1.0 +[May 1, 2026 - 07:18:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 5069.9487iter count: 1.0 +[May 1, 2026 - 07:18:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.3196 iter count: 1.0 +[May 1, 2026 - 07:18:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 41742.0820iter count: 1.0 +[May 1, 2026 - 07:18:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 0.9441 iter count: 1.0 +[May 1, 2026 - 07:18:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 6.6073 iter count: 1.0 +[May 1, 2026 - 07:18:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 1, 2026 - 07:18:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:18:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 07:19:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.660623 41.817497 41.529205 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.852539 67.953125 67.550781 +util 95.250000 99.000000 91.000000 +[May 1, 2026 - 07:19:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 456 : avg iteration time 47.88 seconds +[May 1, 2026 - 07:19:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 456 : data loading time 0.00 +[May 1, 2026 - 07:19:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 456 : avg forward pass time 13.01 +[May 1, 2026 - 07:19:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 456 : backward pass time 10.15 +[May 1, 2026 - 07:19:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 456 : optimizer step time 1.57 +[May 1, 2026 - 07:19:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 456-------------------- +[May 1, 2026 - 07:19:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0121 iter count: 1.0 +[May 1, 2026 - 07:19:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[May 1, 2026 - 07:19:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:19:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 07:19:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.661199 41.818966 41.529453 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.018555 68.039062 68.011719 +util 97.500000 100.000000 94.000000 +[May 1, 2026 - 07:19:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 457 : avg iteration time 49.08 seconds +[May 1, 2026 - 07:19:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 457 : data loading time 0.00 +[May 1, 2026 - 07:19:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 457 : avg forward pass time 13.60 +[May 1, 2026 - 07:19:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 457 : backward pass time 10.15 +[May 1, 2026 - 07:19:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 457 : optimizer step time 1.58 +[May 1, 2026 - 07:19:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 457-------------------- +[May 1, 2026 - 07:19:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 1, 2026 - 07:19:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 1, 2026 - 07:19:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:19:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 07:20:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.661715 41.819927 41.529587 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.023438 68.058594 68.011719 +util 93.500000 99.000000 88.000000 +[May 1, 2026 - 07:20:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 458 : avg iteration time 49.04 seconds +[May 1, 2026 - 07:20:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 458 : data loading time 0.00 +[May 1, 2026 - 07:20:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 458 : avg forward pass time 13.58 +[May 1, 2026 - 07:20:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 458 : backward pass time 10.15 +[May 1, 2026 - 07:20:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 458 : optimizer step time 1.57 +[May 1, 2026 - 07:20:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 458-------------------- +[May 1, 2026 - 07:20:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 07:20:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 07:20:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:20:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 07:21:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.662013 41.820362 41.530128 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.028320 68.078125 68.011719 +util 92.750000 99.000000 87.000000 +[May 1, 2026 - 07:21:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 459 : avg iteration time 49.00 seconds +[May 1, 2026 - 07:21:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 459 : data loading time 0.00 +[May 1, 2026 - 07:21:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 459 : avg forward pass time 13.56 +[May 1, 2026 - 07:21:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 459 : backward pass time 10.14 +[May 1, 2026 - 07:21:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 459 : optimizer step time 1.58 +[May 1, 2026 - 07:21:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 459-------------------- +[May 1, 2026 - 07:21:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 07:21:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 07:21:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:21:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 07:24:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.662015 41.819065 41.530445 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.041016 136.076172 136.029297 +util 63.000000 67.000000 58.000000 +[May 1, 2026 - 07:24:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 460 : avg iteration time 165.92 seconds +[May 1, 2026 - 07:24:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 460 : data loading time 0.00 +[May 1, 2026 - 07:24:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 460 : avg forward pass time 74.33 +[May 1, 2026 - 07:24:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 460 : backward pass time 8.42 +[May 1, 2026 - 07:24:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 460 : optimizer step time 0.40 +[May 1, 2026 - 07:24:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 460-------------------- +[May 1, 2026 - 07:24:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 465789.8691 iter count: 1.0 +[May 1, 2026 - 07:24:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5170 iter count: 1.0 +[May 1, 2026 - 07:24:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 465789.8691iter count: 1.0 +[May 1, 2026 - 07:24:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:24:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 5.0011iter count: 1.0 +[May 1, 2026 - 07:24:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.4634 iter count: 1.0 +[May 1, 2026 - 07:24:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.0750 iter count: 1.0 +[May 1, 2026 - 07:24:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 926869.3750iter count: 1.0 +[May 1, 2026 - 07:24:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 2.5267 iter count: 1.0 +[May 1, 2026 - 07:24:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 10324537.0000iter count: 1.0 +[May 1, 2026 - 07:24:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5170iter count: 1.0 +[May 1, 2026 - 07:24:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 7.0693 iter count: 1.0 +[May 1, 2026 - 07:24:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 1381513.2500iter count: 1.0 +[May 1, 2026 - 07:24:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.8995 iter count: 1.0 +[May 1, 2026 - 07:24:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 18014.3418iter count: 1.0 +[May 1, 2026 - 07:24:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.6066 iter count: 1.0 +[May 1, 2026 - 07:24:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 74107.7891iter count: 1.0 +[May 1, 2026 - 07:24:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 8.0750 iter count: 1.0 +[May 1, 2026 - 07:24:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 10324537.0000iter count: 1.0 +[May 1, 2026 - 07:24:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 07:24:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:24:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 07:25:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.662366 41.819088 41.530491 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.740234 67.933594 67.160156 +util 95.750000 100.000000 91.000000 +[May 1, 2026 - 07:25:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 461 : avg iteration time 45.13 seconds +[May 1, 2026 - 07:25:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 461 : data loading time 0.00 +[May 1, 2026 - 07:25:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 461 : avg forward pass time 11.63 +[May 1, 2026 - 07:25:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 461 : backward pass time 10.13 +[May 1, 2026 - 07:25:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 461 : optimizer step time 1.58 +[May 1, 2026 - 07:25:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 461-------------------- +[May 1, 2026 - 07:25:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0115 iter count: 1.0 +[May 1, 2026 - 07:25:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[May 1, 2026 - 07:25:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:25:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 07:25:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.662894 41.820675 41.530796 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.857422 67.972656 67.511719 +util 96.250000 100.000000 91.000000 +[May 1, 2026 - 07:25:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 462 : avg iteration time 43.49 seconds +[May 1, 2026 - 07:25:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 462 : data loading time 0.00 +[May 1, 2026 - 07:25:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 462 : avg forward pass time 10.84 +[May 1, 2026 - 07:25:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 462 : backward pass time 10.11 +[May 1, 2026 - 07:25:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 462 : optimizer step time 1.58 +[May 1, 2026 - 07:25:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 462-------------------- +[May 1, 2026 - 07:25:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 1, 2026 - 07:25:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 1, 2026 - 07:25:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:25:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 07:26:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.663389 41.821224 41.531807 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.706055 67.757812 67.550781 +util 94.250000 99.000000 89.000000 +[May 1, 2026 - 07:26:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 463 : avg iteration time 46.27 seconds +[May 1, 2026 - 07:26:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 463 : data loading time 0.00 +[May 1, 2026 - 07:26:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 463 : avg forward pass time 12.25 +[May 1, 2026 - 07:26:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 463 : backward pass time 10.10 +[May 1, 2026 - 07:26:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 463 : optimizer step time 1.55 +[May 1, 2026 - 07:26:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 463-------------------- +[May 1, 2026 - 07:26:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 1, 2026 - 07:26:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 1, 2026 - 07:26:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:26:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 07:27:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.663565 41.821609 41.531925 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.974609 68.039062 67.953125 +util 94.500000 99.000000 89.000000 +[May 1, 2026 - 07:27:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 464 : avg iteration time 46.30 seconds +[May 1, 2026 - 07:27:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 464 : data loading time 0.00 +[May 1, 2026 - 07:27:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 464 : avg forward pass time 12.24 +[May 1, 2026 - 07:27:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 464 : backward pass time 10.12 +[May 1, 2026 - 07:27:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 464 : optimizer step time 1.57 +[May 1, 2026 - 07:27:21 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 464-------------------- +[May 1, 2026 - 07:27:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0101 iter count: 1.0 +[May 1, 2026 - 07:27:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 1, 2026 - 07:27:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:27:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 07:30:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.663079 41.819420 41.532036 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.080078 136.087891 136.056641 +util 62.250000 69.000000 56.000000 +[May 1, 2026 - 07:30:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 465 : avg iteration time 165.70 seconds +[May 1, 2026 - 07:30:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 465 : data loading time 0.00 +[May 1, 2026 - 07:30:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 465 : avg forward pass time 74.24 +[May 1, 2026 - 07:30:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 465 : backward pass time 8.43 +[May 1, 2026 - 07:30:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 465 : optimizer step time 0.39 +[May 1, 2026 - 07:30:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 465-------------------- +[May 1, 2026 - 07:30:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 57387.5225 iter count: 1.0 +[May 1, 2026 - 07:30:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4920 iter count: 1.0 +[May 1, 2026 - 07:30:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 57387.5225iter count: 1.0 +[May 1, 2026 - 07:30:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:30:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.6182iter count: 1.0 +[May 1, 2026 - 07:30:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.2568 iter count: 1.0 +[May 1, 2026 - 07:30:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.0372 iter count: 1.0 +[May 1, 2026 - 07:30:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 124543.8281iter count: 1.0 +[May 1, 2026 - 07:30:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 12.3504iter count: 1.0 +[May 1, 2026 - 07:30:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 1295501.6250iter count: 1.0 +[May 1, 2026 - 07:30:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4920iter count: 1.0 +[May 1, 2026 - 07:30:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 1.2568 iter count: 1.0 +[May 1, 2026 - 07:30:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 12.3504 iter count: 1.0 +[May 1, 2026 - 07:30:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.0317 iter count: 1.0 +[May 1, 2026 - 07:30:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 3175.8533iter count: 1.0 +[May 1, 2026 - 07:30:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.4647 iter count: 1.0 +[May 1, 2026 - 07:30:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 55797.9805iter count: 1.0 +[May 1, 2026 - 07:30:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.9273 iter count: 1.0 +[May 1, 2026 - 07:30:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 140738.6719iter count: 1.0 +[May 1, 2026 - 07:30:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 07:30:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:30:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 07:30:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.663226 41.819767 41.532280 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.832520 67.939453 67.796875 +util 97.000000 100.000000 93.000000 +[May 1, 2026 - 07:30:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 466 : avg iteration time 45.15 seconds +[May 1, 2026 - 07:30:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 466 : data loading time 0.00 +[May 1, 2026 - 07:30:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 466 : avg forward pass time 11.66 +[May 1, 2026 - 07:30:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 466 : backward pass time 10.13 +[May 1, 2026 - 07:30:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 466 : optimizer step time 1.57 +[May 1, 2026 - 07:30:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 466-------------------- +[May 1, 2026 - 07:30:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0121 iter count: 1.0 +[May 1, 2026 - 07:30:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[May 1, 2026 - 07:30:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:30:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 07:31:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.663748 41.820866 41.533150 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.686035 67.757812 67.470703 +util 96.000000 99.000000 91.000000 +[May 1, 2026 - 07:31:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 467 : avg iteration time 48.99 seconds +[May 1, 2026 - 07:31:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 467 : data loading time 0.00 +[May 1, 2026 - 07:31:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 467 : avg forward pass time 13.58 +[May 1, 2026 - 07:31:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 467 : backward pass time 10.14 +[May 1, 2026 - 07:31:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 467 : optimizer step time 1.57 +[May 1, 2026 - 07:31:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 467-------------------- +[May 1, 2026 - 07:31:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0111 iter count: 1.0 +[May 1, 2026 - 07:31:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 1, 2026 - 07:31:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:31:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 07:32:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.664081 41.821121 41.534119 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.827637 67.933594 67.509766 +util 96.000000 99.000000 91.000000 +[May 1, 2026 - 07:32:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 468 : avg iteration time 49.00 seconds +[May 1, 2026 - 07:32:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 468 : data loading time 0.00 +[May 1, 2026 - 07:32:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 468 : avg forward pass time 13.57 +[May 1, 2026 - 07:32:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 468 : backward pass time 10.15 +[May 1, 2026 - 07:32:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 468 : optimizer step time 1.58 +[May 1, 2026 - 07:32:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 468-------------------- +[May 1, 2026 - 07:32:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0104 iter count: 1.0 +[May 1, 2026 - 07:32:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 1, 2026 - 07:32:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:32:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 07:33:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.664168 41.821228 41.534130 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.495605 67.503906 67.470703 +util 94.000000 99.000000 89.000000 +[May 1, 2026 - 07:33:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 469 : avg iteration time 46.33 seconds +[May 1, 2026 - 07:33:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 469 : data loading time 0.00 +[May 1, 2026 - 07:33:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 469 : avg forward pass time 12.23 +[May 1, 2026 - 07:33:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 469 : backward pass time 10.12 +[May 1, 2026 - 07:33:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 469 : optimizer step time 1.57 +[May 1, 2026 - 07:33:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 469-------------------- +[May 1, 2026 - 07:33:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 07:33:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 07:33:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:33:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 07:35:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.664358 41.820332 41.534126 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.055664 136.076172 136.048828 +util 64.750000 69.000000 60.000000 +[May 1, 2026 - 07:35:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 470 : avg iteration time 160.67 seconds +[May 1, 2026 - 07:35:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 470 : data loading time 0.00 +[May 1, 2026 - 07:35:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 470 : avg forward pass time 71.72 +[May 1, 2026 - 07:35:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 470 : backward pass time 8.45 +[May 1, 2026 - 07:35:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 470 : optimizer step time 0.40 +[May 1, 2026 - 07:35:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 470-------------------- +[May 1, 2026 - 07:35:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 225183.7598 iter count: 1.0 +[May 1, 2026 - 07:35:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5328 iter count: 1.0 +[May 1, 2026 - 07:35:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 225183.7598iter count: 1.0 +[May 1, 2026 - 07:35:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:35:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.8492iter count: 1.0 +[May 1, 2026 - 07:35:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.0053 iter count: 1.0 +[May 1, 2026 - 07:35:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.7502 iter count: 1.0 +[May 1, 2026 - 07:35:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 468767.0000iter count: 1.0 +[May 1, 2026 - 07:35:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 7.4671 iter count: 1.0 +[May 1, 2026 - 07:35:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 5391898.5000iter count: 1.0 +[May 1, 2026 - 07:35:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5328iter count: 1.0 +[May 1, 2026 - 07:35:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 7.0283 iter count: 1.0 +[May 1, 2026 - 07:35:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 1272740.0000iter count: 1.0 +[May 1, 2026 - 07:35:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.3905 iter count: 1.0 +[May 1, 2026 - 07:35:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 48098.7578iter count: 1.0 +[May 1, 2026 - 07:35:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 6.3461 iter count: 1.0 +[May 1, 2026 - 07:35:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 325205.9688iter count: 1.0 +[May 1, 2026 - 07:35:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 1.6718 iter count: 1.0 +[May 1, 2026 - 07:35:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 28.3206 iter count: 1.0 +[May 1, 2026 - 07:35:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 1, 2026 - 07:35:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:35:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 07:36:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.664671 41.820545 41.534340 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.862305 67.972656 67.531250 +util 95.500000 100.000000 91.000000 +[May 1, 2026 - 07:36:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 471 : avg iteration time 45.10 seconds +[May 1, 2026 - 07:36:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 471 : data loading time 0.00 +[May 1, 2026 - 07:36:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 471 : avg forward pass time 11.62 +[May 1, 2026 - 07:36:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 471 : backward pass time 10.14 +[May 1, 2026 - 07:36:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 471 : optimizer step time 1.58 +[May 1, 2026 - 07:36:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 471-------------------- +[May 1, 2026 - 07:36:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0142 iter count: 1.0 +[May 1, 2026 - 07:36:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0142 iter count: 1.0 +[May 1, 2026 - 07:36:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:36:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 07:37:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.665133 41.821365 41.534355 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.842773 67.972656 67.453125 +util 92.500000 99.000000 87.000000 +[May 1, 2026 - 07:37:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 472 : avg iteration time 48.97 seconds +[May 1, 2026 - 07:37:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 472 : data loading time 0.00 +[May 1, 2026 - 07:37:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 472 : avg forward pass time 13.56 +[May 1, 2026 - 07:37:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 472 : backward pass time 10.14 +[May 1, 2026 - 07:37:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 472 : optimizer step time 1.59 +[May 1, 2026 - 07:37:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 472-------------------- +[May 1, 2026 - 07:37:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0111 iter count: 1.0 +[May 1, 2026 - 07:37:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 1, 2026 - 07:37:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:37:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 07:38:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.665833 41.823193 41.534367 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.979492 68.000000 67.972656 +util 96.000000 99.000000 91.000000 +[May 1, 2026 - 07:38:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 473 : avg iteration time 46.34 seconds +[May 1, 2026 - 07:38:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 473 : data loading time 0.00 +[May 1, 2026 - 07:38:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 473 : avg forward pass time 12.24 +[May 1, 2026 - 07:38:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 473 : backward pass time 10.14 +[May 1, 2026 - 07:38:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 473 : optimizer step time 1.57 +[May 1, 2026 - 07:38:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 473-------------------- +[May 1, 2026 - 07:38:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 1, 2026 - 07:38:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 1, 2026 - 07:38:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:38:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 07:39:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666067 41.823978 41.534393 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.979492 68.000000 67.972656 +util 94.000000 99.000000 89.000000 +[May 1, 2026 - 07:39:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 474 : avg iteration time 48.93 seconds +[May 1, 2026 - 07:39:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 474 : data loading time 0.00 +[May 1, 2026 - 07:39:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 474 : avg forward pass time 13.53 +[May 1, 2026 - 07:39:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 474 : backward pass time 10.14 +[May 1, 2026 - 07:39:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 474 : optimizer step time 1.58 +[May 1, 2026 - 07:39:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 474-------------------- +[May 1, 2026 - 07:39:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 07:39:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 07:39:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:39:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 07:41:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.665460 41.821434 41.534389 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.057129 136.076172 136.050781 +util 65.250000 72.000000 60.000000 +[May 1, 2026 - 07:41:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 475 : avg iteration time 160.42 seconds +[May 1, 2026 - 07:41:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 475 : data loading time 0.00 +[May 1, 2026 - 07:41:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 475 : avg forward pass time 71.58 +[May 1, 2026 - 07:41:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 475 : backward pass time 8.42 +[May 1, 2026 - 07:41:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 475 : optimizer step time 0.40 +[May 1, 2026 - 07:41:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 475-------------------- +[May 1, 2026 - 07:41:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 320095.4917 iter count: 1.0 +[May 1, 2026 - 07:41:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5218 iter count: 1.0 +[May 1, 2026 - 07:41:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 320095.4917iter count: 1.0 +[May 1, 2026 - 07:41:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:41:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.2664iter count: 1.0 +[May 1, 2026 - 07:41:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.5263 iter count: 1.0 +[May 1, 2026 - 07:41:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.9290 iter count: 1.0 +[May 1, 2026 - 07:41:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 626271.9375iter count: 1.0 +[May 1, 2026 - 07:41:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 2.8653 iter count: 1.0 +[May 1, 2026 - 07:41:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 7709056.5000iter count: 1.0 +[May 1, 2026 - 07:41:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5218iter count: 1.0 +[May 1, 2026 - 07:41:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 6.1109 iter count: 1.0 +[May 1, 2026 - 07:41:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 203180.7812iter count: 1.0 +[May 1, 2026 - 07:41:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.5518 iter count: 1.0 +[May 1, 2026 - 07:41:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 1216.2563iter count: 1.0 +[May 1, 2026 - 07:41:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.8685 iter count: 1.0 +[May 1, 2026 - 07:41:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 2291.7742iter count: 1.0 +[May 1, 2026 - 07:41:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.2433 iter count: 1.0 +[May 1, 2026 - 07:41:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 656.2550iter count: 1.0 +[May 1, 2026 - 07:41:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 1, 2026 - 07:41:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:41:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 07:42:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.665581 41.821613 41.534512 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.862305 67.972656 67.531250 +util 93.500000 100.000000 88.000000 +[May 1, 2026 - 07:42:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 476 : avg iteration time 47.77 seconds +[May 1, 2026 - 07:42:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 476 : data loading time 0.00 +[May 1, 2026 - 07:42:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 476 : avg forward pass time 12.96 +[May 1, 2026 - 07:42:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 476 : backward pass time 10.14 +[May 1, 2026 - 07:42:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 476 : optimizer step time 1.58 +[May 1, 2026 - 07:42:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 476-------------------- +[May 1, 2026 - 07:42:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0123 iter count: 1.0 +[May 1, 2026 - 07:42:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0123 iter count: 1.0 +[May 1, 2026 - 07:42:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:42:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 07:43:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.665879 41.822681 41.534515 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.023438 68.031250 68.000000 +util 97.250000 100.000000 93.000000 +[May 1, 2026 - 07:43:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 477 : avg iteration time 46.30 seconds +[May 1, 2026 - 07:43:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 477 : data loading time 0.00 +[May 1, 2026 - 07:43:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 477 : avg forward pass time 12.22 +[May 1, 2026 - 07:43:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 477 : backward pass time 10.16 +[May 1, 2026 - 07:43:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 477 : optimizer step time 1.57 +[May 1, 2026 - 07:43:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 477-------------------- +[May 1, 2026 - 07:43:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 07:43:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 07:43:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:43:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 07:44:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666265 41.824203 41.534519 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.735352 67.816406 67.492188 +util 95.750000 99.000000 91.000000 +[May 1, 2026 - 07:44:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 478 : avg iteration time 46.37 seconds +[May 1, 2026 - 07:44:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 478 : data loading time 0.00 +[May 1, 2026 - 07:44:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 478 : avg forward pass time 12.25 +[May 1, 2026 - 07:44:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 478 : backward pass time 10.14 +[May 1, 2026 - 07:44:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 478 : optimizer step time 1.58 +[May 1, 2026 - 07:44:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 478-------------------- +[May 1, 2026 - 07:44:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0114 iter count: 1.0 +[May 1, 2026 - 07:44:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 1, 2026 - 07:44:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:44:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 07:44:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666262 41.824181 41.534523 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.745117 67.816406 67.531250 +util 93.500000 98.000000 88.000000 +[May 1, 2026 - 07:44:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 479 : avg iteration time 48.97 seconds +[May 1, 2026 - 07:44:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 479 : data loading time 0.00 +[May 1, 2026 - 07:44:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 479 : avg forward pass time 13.55 +[May 1, 2026 - 07:44:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 479 : backward pass time 10.14 +[May 1, 2026 - 07:44:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 479 : optimizer step time 1.57 +[May 1, 2026 - 07:44:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 479-------------------- +[May 1, 2026 - 07:44:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 07:44:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 07:44:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:44:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 07:47:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.665811 41.822170 41.534672 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.045898 136.095703 136.029297 +util 63.000000 70.000000 57.000000 +[May 1, 2026 - 07:47:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 480 : avg iteration time 165.72 seconds +[May 1, 2026 - 07:47:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 480 : data loading time 0.00 +[May 1, 2026 - 07:47:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 480 : avg forward pass time 74.25 +[May 1, 2026 - 07:47:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 480 : backward pass time 8.40 +[May 1, 2026 - 07:47:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 480 : optimizer step time 0.40 +[May 1, 2026 - 07:47:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 480-------------------- +[May 1, 2026 - 07:47:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 3172831.1252 iter count: 1.0 +[May 1, 2026 - 07:47:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4792 iter count: 1.0 +[May 1, 2026 - 07:47:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 3172831.1252iter count: 1.0 +[May 1, 2026 - 07:47:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:47:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 5.0370iter count: 1.0 +[May 1, 2026 - 07:47:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.5476 iter count: 1.0 +[May 1, 2026 - 07:47:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 9.7161 iter count: 1.0 +[May 1, 2026 - 07:47:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 8770485.0000iter count: 1.0 +[May 1, 2026 - 07:47:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 2.9897 iter count: 1.0 +[May 1, 2026 - 07:47:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 274964288.0000iter count: 1.0 +[May 1, 2026 - 07:47:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4792iter count: 1.0 +[May 1, 2026 - 07:47:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.7707 iter count: 1.0 +[May 1, 2026 - 07:47:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 1884.6443iter count: 1.0 +[May 1, 2026 - 07:47:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 6.2881 iter count: 1.0 +[May 1, 2026 - 07:47:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 289581.7188iter count: 1.0 +[May 1, 2026 - 07:47:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.2406 iter count: 1.0 +[May 1, 2026 - 07:47:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 35639.7344iter count: 1.0 +[May 1, 2026 - 07:47:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 7.1066 iter count: 1.0 +[May 1, 2026 - 07:47:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 1488504.6250iter count: 1.0 +[May 1, 2026 - 07:47:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 07:47:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:47:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 07:48:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.665863 41.822182 41.534695 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.003906 68.011719 67.980469 +util 97.750000 100.000000 94.000000 +[May 1, 2026 - 07:48:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 481 : avg iteration time 47.76 seconds +[May 1, 2026 - 07:48:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 481 : data loading time 0.00 +[May 1, 2026 - 07:48:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 481 : avg forward pass time 12.94 +[May 1, 2026 - 07:48:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 481 : backward pass time 10.15 +[May 1, 2026 - 07:48:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 481 : optimizer step time 1.56 +[May 1, 2026 - 07:48:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 481-------------------- +[May 1, 2026 - 07:48:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0108 iter count: 1.0 +[May 1, 2026 - 07:48:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 1, 2026 - 07:48:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:48:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 07:49:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666334 41.823921 41.534718 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.042969 68.136719 68.011719 +util 95.500000 99.000000 90.000000 +[May 1, 2026 - 07:49:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 482 : avg iteration time 48.95 seconds +[May 1, 2026 - 07:49:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 482 : data loading time 0.00 +[May 1, 2026 - 07:49:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 482 : avg forward pass time 13.54 +[May 1, 2026 - 07:49:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 482 : backward pass time 10.13 +[May 1, 2026 - 07:49:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 482 : optimizer step time 1.58 +[May 1, 2026 - 07:49:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 482-------------------- +[May 1, 2026 - 07:49:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 07:49:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 07:49:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:49:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 07:50:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666470 41.824429 41.534767 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.891602 68.011719 67.531250 +util 95.750000 100.000000 91.000000 +[May 1, 2026 - 07:50:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 483 : avg iteration time 48.96 seconds +[May 1, 2026 - 07:50:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 483 : data loading time 0.00 +[May 1, 2026 - 07:50:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 483 : avg forward pass time 13.56 +[May 1, 2026 - 07:50:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 483 : backward pass time 10.15 +[May 1, 2026 - 07:50:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 483 : optimizer step time 1.56 +[May 1, 2026 - 07:50:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 483-------------------- +[May 1, 2026 - 07:50:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 07:50:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 07:50:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:50:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 07:50:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666568 41.824684 41.534786 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.901367 68.011719 67.570312 +util 94.500000 99.000000 90.000000 +[May 1, 2026 - 07:50:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 484 : avg iteration time 43.58 seconds +[May 1, 2026 - 07:50:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 484 : data loading time 0.00 +[May 1, 2026 - 07:50:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 484 : avg forward pass time 10.86 +[May 1, 2026 - 07:50:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 484 : backward pass time 10.15 +[May 1, 2026 - 07:50:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 484 : optimizer step time 1.57 +[May 1, 2026 - 07:50:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 484-------------------- +[May 1, 2026 - 07:50:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0108 iter count: 1.0 +[May 1, 2026 - 07:50:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 1, 2026 - 07:50:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:50:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 07:53:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.665992 41.822327 41.534801 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.026367 136.029297 136.017578 +util 61.250000 69.000000 55.000000 +[May 1, 2026 - 07:53:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 485 : avg iteration time 165.91 seconds +[May 1, 2026 - 07:53:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 485 : data loading time 0.00 +[May 1, 2026 - 07:53:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 485 : avg forward pass time 74.35 +[May 1, 2026 - 07:53:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 485 : backward pass time 8.43 +[May 1, 2026 - 07:53:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 485 : optimizer step time 0.39 +[May 1, 2026 - 07:53:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 485-------------------- +[May 1, 2026 - 07:53:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 219783.4307 iter count: 1.0 +[May 1, 2026 - 07:53:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6014 iter count: 1.0 +[May 1, 2026 - 07:53:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 219783.4307iter count: 1.0 +[May 1, 2026 - 07:53:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:53:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.4646iter count: 1.0 +[May 1, 2026 - 07:53:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.2956 iter count: 1.0 +[May 1, 2026 - 07:53:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.8432 iter count: 1.0 +[May 1, 2026 - 07:53:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 343525.0938iter count: 1.0 +[May 1, 2026 - 07:53:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 13.3445iter count: 1.0 +[May 1, 2026 - 07:53:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 6493979.5000iter count: 1.0 +[May 1, 2026 - 07:53:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6014iter count: 1.0 +[May 1, 2026 - 07:53:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 1.4069 iter count: 1.0 +[May 1, 2026 - 07:53:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 16.6723 iter count: 1.0 +[May 1, 2026 - 07:53:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.9469 iter count: 1.0 +[May 1, 2026 - 07:53:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 146364.5938iter count: 1.0 +[May 1, 2026 - 07:53:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.8908 iter count: 1.0 +[May 1, 2026 - 07:53:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 130823.7969iter count: 1.0 +[May 1, 2026 - 07:53:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.4931 iter count: 1.0 +[May 1, 2026 - 07:53:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 436356.8125iter count: 1.0 +[May 1, 2026 - 07:53:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[May 1, 2026 - 07:53:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:53:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 07:54:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666002 41.822334 41.534836 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.574219 67.621094 67.433594 +util 96.000000 99.000000 91.000000 +[May 1, 2026 - 07:54:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 486 : avg iteration time 45.16 seconds +[May 1, 2026 - 07:54:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 486 : data loading time 0.00 +[May 1, 2026 - 07:54:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 486 : avg forward pass time 11.66 +[May 1, 2026 - 07:54:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 486 : backward pass time 10.13 +[May 1, 2026 - 07:54:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 486 : optimizer step time 1.57 +[May 1, 2026 - 07:54:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 486-------------------- +[May 1, 2026 - 07:54:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0108 iter count: 1.0 +[May 1, 2026 - 07:54:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 1, 2026 - 07:54:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:54:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 07:55:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666302 41.823322 41.534954 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.130859 68.207031 67.902344 +util 95.000000 99.000000 89.000000 +[May 1, 2026 - 07:55:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 487 : avg iteration time 48.97 seconds +[May 1, 2026 - 07:55:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 487 : data loading time 0.00 +[May 1, 2026 - 07:55:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 487 : avg forward pass time 13.55 +[May 1, 2026 - 07:55:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 487 : backward pass time 10.17 +[May 1, 2026 - 07:55:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 487 : optimizer step time 1.55 +[May 1, 2026 - 07:55:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 487-------------------- +[May 1, 2026 - 07:55:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[May 1, 2026 - 07:55:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[May 1, 2026 - 07:55:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:55:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 07:55:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666433 41.823792 41.534931 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.165039 68.226562 67.980469 +util 95.250000 99.000000 91.000000 +[May 1, 2026 - 07:55:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 488 : avg iteration time 46.37 seconds +[May 1, 2026 - 07:55:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 488 : data loading time 0.00 +[May 1, 2026 - 07:55:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 488 : avg forward pass time 12.28 +[May 1, 2026 - 07:55:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 488 : backward pass time 10.12 +[May 1, 2026 - 07:55:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 488 : optimizer step time 1.57 +[May 1, 2026 - 07:55:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 488-------------------- +[May 1, 2026 - 07:55:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 07:55:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 07:55:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:55:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 07:56:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666567 41.824238 41.534969 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.150391 68.226562 67.921875 +util 95.500000 100.000000 90.000000 +[May 1, 2026 - 07:56:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 489 : avg iteration time 46.24 seconds +[May 1, 2026 - 07:56:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 489 : data loading time 0.00 +[May 1, 2026 - 07:56:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 489 : avg forward pass time 12.20 +[May 1, 2026 - 07:56:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 489 : backward pass time 10.12 +[May 1, 2026 - 07:56:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 489 : optimizer step time 1.58 +[May 1, 2026 - 07:56:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 489-------------------- +[May 1, 2026 - 07:56:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 07:56:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 07:56:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:56:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 07:59:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666057 41.822178 41.534981 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.041016 136.193359 135.990234 +util 63.250000 71.000000 58.000000 +[May 1, 2026 - 07:59:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 490 : avg iteration time 168.70 seconds +[May 1, 2026 - 07:59:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 490 : data loading time 0.00 +[May 1, 2026 - 07:59:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 490 : avg forward pass time 75.71 +[May 1, 2026 - 07:59:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 490 : backward pass time 8.46 +[May 1, 2026 - 07:59:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 490 : optimizer step time 0.40 +[May 1, 2026 - 07:59:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 490-------------------- +[May 1, 2026 - 07:59:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 167355.5000 iter count: 1.0 +[May 1, 2026 - 07:59:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5726 iter count: 1.0 +[May 1, 2026 - 07:59:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 167355.5000iter count: 1.0 +[May 1, 2026 - 07:59:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:59:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.9851iter count: 1.0 +[May 1, 2026 - 07:59:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.4981 iter count: 1.0 +[May 1, 2026 - 07:59:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.6641 iter count: 1.0 +[May 1, 2026 - 07:59:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 315381.5625iter count: 1.0 +[May 1, 2026 - 07:59:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 20.0085iter count: 1.0 +[May 1, 2026 - 07:59:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 4539304.0000iter count: 1.0 +[May 1, 2026 - 07:59:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5726iter count: 1.0 +[May 1, 2026 - 07:59:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.7676 iter count: 1.0 +[May 1, 2026 - 07:59:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 102244.2734iter count: 1.0 +[May 1, 2026 - 07:59:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 6.1796 iter count: 1.0 +[May 1, 2026 - 07:59:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 233114.5781iter count: 1.0 +[May 1, 2026 - 07:59:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 6.5078 iter count: 1.0 +[May 1, 2026 - 07:59:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 449331.5312iter count: 1.0 +[May 1, 2026 - 07:59:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.8977 iter count: 1.0 +[May 1, 2026 - 07:59:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 132639.1094iter count: 1.0 +[May 1, 2026 - 07:59:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 07:59:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 07:59:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 08:00:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666059 41.822166 41.534958 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.828125 67.855469 67.746094 +util 96.000000 99.000000 92.000000 +[May 1, 2026 - 08:00:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 491 : avg iteration time 47.87 seconds +[May 1, 2026 - 08:00:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 491 : data loading time 0.00 +[May 1, 2026 - 08:00:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 491 : avg forward pass time 13.02 +[May 1, 2026 - 08:00:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 491 : backward pass time 10.13 +[May 1, 2026 - 08:00:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 491 : optimizer step time 1.56 +[May 1, 2026 - 08:00:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 491-------------------- +[May 1, 2026 - 08:00:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[May 1, 2026 - 08:00:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 1, 2026 - 08:00:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:00:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 08:01:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666473 41.823681 41.534954 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.003906 68.011719 67.980469 +util 93.250000 100.000000 88.000000 +[May 1, 2026 - 08:01:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 492 : avg iteration time 46.31 seconds +[May 1, 2026 - 08:01:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 492 : data loading time 0.00 +[May 1, 2026 - 08:01:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 492 : avg forward pass time 12.25 +[May 1, 2026 - 08:01:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 492 : backward pass time 10.10 +[May 1, 2026 - 08:01:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 492 : optimizer step time 1.56 +[May 1, 2026 - 08:01:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 492-------------------- +[May 1, 2026 - 08:01:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 08:01:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 08:01:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:01:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 08:01:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666751 41.824677 41.535049 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.784180 67.796875 67.746094 +util 93.000000 99.000000 88.000000 +[May 1, 2026 - 08:01:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 493 : avg iteration time 46.25 seconds +[May 1, 2026 - 08:01:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 493 : data loading time 0.00 +[May 1, 2026 - 08:01:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 493 : avg forward pass time 12.19 +[May 1, 2026 - 08:01:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 493 : backward pass time 10.14 +[May 1, 2026 - 08:01:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 493 : optimizer step time 1.58 +[May 1, 2026 - 08:01:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 493-------------------- +[May 1, 2026 - 08:01:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0117 iter count: 1.0 +[May 1, 2026 - 08:01:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[May 1, 2026 - 08:01:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:01:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 08:02:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666792 41.824703 41.535057 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.833008 67.941406 67.796875 +util 95.250000 99.000000 91.000000 +[May 1, 2026 - 08:02:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 494 : avg iteration time 46.23 seconds +[May 1, 2026 - 08:02:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 494 : data loading time 0.00 +[May 1, 2026 - 08:02:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 494 : avg forward pass time 12.19 +[May 1, 2026 - 08:02:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 494 : backward pass time 10.13 +[May 1, 2026 - 08:02:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 494 : optimizer step time 1.57 +[May 1, 2026 - 08:02:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 494-------------------- +[May 1, 2026 - 08:02:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 08:02:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 08:02:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:02:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 08:05:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666246 41.822536 41.535088 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 135.987305 136.037109 135.970703 +util 64.500000 69.000000 59.000000 +[May 1, 2026 - 08:05:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 495 : avg iteration time 166.27 seconds +[May 1, 2026 - 08:05:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 495 : data loading time 0.00 +[May 1, 2026 - 08:05:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 495 : avg forward pass time 74.53 +[May 1, 2026 - 08:05:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 495 : backward pass time 8.45 +[May 1, 2026 - 08:05:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 495 : optimizer step time 0.40 +[May 1, 2026 - 08:05:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 495-------------------- +[May 1, 2026 - 08:05:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 214352.0771 iter count: 1.0 +[May 1, 2026 - 08:05:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6053 iter count: 1.0 +[May 1, 2026 - 08:05:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 214352.0771iter count: 1.0 +[May 1, 2026 - 08:05:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:05:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.8441iter count: 1.0 +[May 1, 2026 - 08:05:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.5102 iter count: 1.0 +[May 1, 2026 - 08:05:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.6537 iter count: 1.0 +[May 1, 2026 - 08:05:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 352378.6250iter count: 1.0 +[May 1, 2026 - 08:05:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 20.5008iter count: 1.0 +[May 1, 2026 - 08:05:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 4445689.5000iter count: 1.0 +[May 1, 2026 - 08:05:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6053iter count: 1.0 +[May 1, 2026 - 08:05:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.6811 iter count: 1.0 +[May 1, 2026 - 08:05:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 86007.2109iter count: 1.0 +[May 1, 2026 - 08:05:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.5494 iter count: 1.0 +[May 1, 2026 - 08:05:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 66087.9375iter count: 1.0 +[May 1, 2026 - 08:05:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 6.2685 iter count: 1.0 +[May 1, 2026 - 08:05:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 278477.9688iter count: 1.0 +[May 1, 2026 - 08:05:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.9426 iter count: 1.0 +[May 1, 2026 - 08:05:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 2657.4702iter count: 1.0 +[May 1, 2026 - 08:05:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 1, 2026 - 08:05:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:05:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 08:06:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666279 41.822628 41.535103 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.881836 67.902344 67.875000 +util 98.000000 100.000000 94.000000 +[May 1, 2026 - 08:06:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 496 : avg iteration time 47.84 seconds +[May 1, 2026 - 08:06:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 496 : data loading time 0.00 +[May 1, 2026 - 08:06:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 496 : avg forward pass time 12.99 +[May 1, 2026 - 08:06:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 496 : backward pass time 10.13 +[May 1, 2026 - 08:06:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 496 : optimizer step time 1.58 +[May 1, 2026 - 08:06:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 496-------------------- +[May 1, 2026 - 08:06:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0112 iter count: 1.0 +[May 1, 2026 - 08:06:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 1, 2026 - 08:06:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:06:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 08:06:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666471 41.823387 41.535107 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.213867 68.304688 67.941406 +util 96.750000 100.000000 92.000000 +[May 1, 2026 - 08:06:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 497 : avg iteration time 43.57 seconds +[May 1, 2026 - 08:06:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 497 : data loading time 0.00 +[May 1, 2026 - 08:06:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 497 : avg forward pass time 10.89 +[May 1, 2026 - 08:06:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 497 : backward pass time 10.10 +[May 1, 2026 - 08:06:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 497 : optimizer step time 1.55 +[May 1, 2026 - 08:06:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 497-------------------- +[May 1, 2026 - 08:06:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 1, 2026 - 08:06:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 1, 2026 - 08:06:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:06:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 08:07:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666604 41.823948 41.535103 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.194336 68.265625 67.980469 +util 95.000000 99.000000 89.000000 +[May 1, 2026 - 08:07:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 498 : avg iteration time 46.37 seconds +[May 1, 2026 - 08:07:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 498 : data loading time 0.00 +[May 1, 2026 - 08:07:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 498 : avg forward pass time 12.25 +[May 1, 2026 - 08:07:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 498 : backward pass time 10.16 +[May 1, 2026 - 08:07:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 498 : optimizer step time 1.57 +[May 1, 2026 - 08:07:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 498-------------------- +[May 1, 2026 - 08:07:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 08:07:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 08:07:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:07:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 08:08:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666615 41.823971 41.535126 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.194336 68.285156 67.921875 +util 97.000000 100.000000 92.000000 +[May 1, 2026 - 08:08:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 499 : avg iteration time 46.24 seconds +[May 1, 2026 - 08:08:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 499 : data loading time 0.00 +[May 1, 2026 - 08:08:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 499 : avg forward pass time 12.20 +[May 1, 2026 - 08:08:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 499 : backward pass time 10.14 +[May 1, 2026 - 08:08:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 499 : optimizer step time 1.56 +[May 1, 2026 - 08:08:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 499-------------------- +[May 1, 2026 - 08:08:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 1, 2026 - 08:08:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 1, 2026 - 08:08:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:08:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 08:11:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666014 41.821537 41.535126 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.032715 136.037109 136.031250 +util 65.000000 72.000000 59.000000 +[May 1, 2026 - 08:11:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 500 : avg iteration time 166.30 seconds +[May 1, 2026 - 08:11:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 500 : data loading time 0.00 +[May 1, 2026 - 08:11:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 500 : avg forward pass time 74.51 +[May 1, 2026 - 08:11:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 500 : backward pass time 8.44 +[May 1, 2026 - 08:11:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 500 : optimizer step time 0.41 +[May 1, 2026 - 08:11:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 500-------------------- +[May 1, 2026 - 08:11:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 608042.8022 iter count: 1.0 +[May 1, 2026 - 08:11:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5438 iter count: 1.0 +[May 1, 2026 - 08:11:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 608042.8022iter count: 1.0 +[May 1, 2026 - 08:11:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:11:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.4797iter count: 1.0 +[May 1, 2026 - 08:11:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.9836 iter count: 1.0 +[May 1, 2026 - 08:11:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.5411 iter count: 1.0 +[May 1, 2026 - 08:11:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 1003213.1250iter count: 1.0 +[May 1, 2026 - 08:11:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 7.1504 iter count: 1.0 +[May 1, 2026 - 08:11:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 26225506.0000iter count: 1.0 +[May 1, 2026 - 08:11:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5438iter count: 1.0 +[May 1, 2026 - 08:11:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 2.5902 iter count: 1.0 +[May 1, 2026 - 08:11:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 177.7568iter count: 1.0 +[May 1, 2026 - 08:11:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 7.5815 iter count: 1.0 +[May 1, 2026 - 08:11:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 3847581.5000iter count: 1.0 +[May 1, 2026 - 08:11:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 2.4499 iter count: 1.0 +[May 1, 2026 - 08:11:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 134.2692iter count: 1.0 +[May 1, 2026 - 08:11:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.1144 iter count: 1.0 +[May 1, 2026 - 08:11:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 507.0971iter count: 1.0 +[May 1, 2026 - 08:11:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 1, 2026 - 08:11:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 1, 2026 - 08:11:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 15.05 seconds +[May 1, 2026 - 08:11:30 | INFO | fastgen.trainer:save_checkpoint:266 ] Saving checkpoint iteration 500 +[May 1, 2026 - 08:11:30 | INFO | fastgen.utils.checkpointer:save:311 ] Saving FSDP model to prefix /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000500 +[May 1, 2026 - 08:12:19 | SUCCESS | fastgen.utils.checkpointer:save:362 ] Model saved at iteration 500 +[May 1, 2026 - 08:12:19 | INFO | fastgen.trainer:validate:358 ] Validation iteration 500 +[May 1, 2026 - 08:12:19 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 08:12:22 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 2.1s +[May 1, 2026 - 08:12:25 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 0: sync_c=6.320 +[May 1, 2026 - 08:12:25 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 08:12:27 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 08:12:30 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 1: sync_c=5.968 +[May 1, 2026 - 08:12:30 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 08:12:32 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 08:12:35 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 2: sync_c=4.882 +[May 1, 2026 - 08:12:35 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 08:12:37 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 08:12:40 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 3: sync_c=5.328 +[May 1, 2026 - 08:12:40 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 08:12:42 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 08:12:45 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 4: sync_c=4.935 +[May 1, 2026 - 08:12:46 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 08:12:47 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 08:12:50 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 5: sync_c=6.618 +[May 1, 2026 - 08:12:51 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 08:12:52 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 08:12:55 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 6: sync_c=6.939 +[May 1, 2026 - 08:12:56 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 08:12:58 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 08:13:01 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 7: sync_c=5.284 +[May 1, 2026 - 08:13:01 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 08:13:03 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 08:13:06 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 8: sync_c=6.000 +[May 1, 2026 - 08:13:06 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:296 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[May 1, 2026 - 08:13:08 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:314 ] [val] AR inference done in 1.8s +[May 1, 2026 - 08:13:11 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 9: sync_c=3.343 +[May 1, 2026 - 08:13:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging val0 stats at iteration 500-------------------- +[May 1, 2026 - 08:13:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0000 iter count: 10.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 1, 2026 - 08:13:27 | INFO | fastgen.callbacks.wandb:on_validation_end:763 ] Logged 10 val videos at iteration 500 +[May 1, 2026 - 08:13:27 | INFO | fastgen.callbacks.wandb:on_validation_end:769 ] [SyncEval] val0 mean sync_c: 5.562 (n=10) +[May 1, 2026 - 08:14:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.667723 41.829193 41.534794 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.101074 69.013672 67.796875 +util 96.000000 99.000000 91.000000 +[May 1, 2026 - 08:14:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 501 : avg iteration time 180.29 seconds +[May 1, 2026 - 08:14:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 501 : data loading time 0.00 +[May 1, 2026 - 08:14:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 501 : avg forward pass time 13.24 +[May 1, 2026 - 08:14:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 501 : backward pass time 10.16 +[May 1, 2026 - 08:14:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 501 : optimizer step time 1.56 +[May 1, 2026 - 08:14:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 501-------------------- +[May 1, 2026 - 08:14:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0101 iter count: 1.0 +[May 1, 2026 - 08:14:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 1, 2026 - 08:14:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:14:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 08:15:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.667407 41.827946 41.534798 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.218262 68.837891 68.011719 +util 93.000000 99.000000 88.000000 +[May 1, 2026 - 08:15:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 502 : avg iteration time 46.52 seconds +[May 1, 2026 - 08:15:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 502 : data loading time 0.00 +[May 1, 2026 - 08:15:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 502 : avg forward pass time 12.28 +[May 1, 2026 - 08:15:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 502 : backward pass time 10.20 +[May 1, 2026 - 08:15:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 502 : optimizer step time 1.58 +[May 1, 2026 - 08:15:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 502-------------------- +[May 1, 2026 - 08:15:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0108 iter count: 1.0 +[May 1, 2026 - 08:15:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 1, 2026 - 08:15:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:15:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 08:15:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.667413 41.827957 41.534805 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.208496 68.798828 68.011719 +util 93.000000 99.000000 88.000000 +[May 1, 2026 - 08:15:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 503 : avg iteration time 46.35 seconds +[May 1, 2026 - 08:15:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 503 : data loading time 0.00 +[May 1, 2026 - 08:15:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 503 : avg forward pass time 12.22 +[May 1, 2026 - 08:15:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 503 : backward pass time 10.17 +[May 1, 2026 - 08:15:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 503 : optimizer step time 1.59 +[May 1, 2026 - 08:15:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 503-------------------- +[May 1, 2026 - 08:15:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 1, 2026 - 08:15:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 1, 2026 - 08:15:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:15:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 08:16:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.667133 41.826801 41.534828 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 68.188965 68.837891 67.972656 +util 94.750000 100.000000 90.000000 +[May 1, 2026 - 08:16:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 504 : avg iteration time 49.03 seconds +[May 1, 2026 - 08:16:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 504 : data loading time 0.00 +[May 1, 2026 - 08:16:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 504 : avg forward pass time 13.57 +[May 1, 2026 - 08:16:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 504 : backward pass time 10.17 +[May 1, 2026 - 08:16:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 504 : optimizer step time 1.57 +[May 1, 2026 - 08:16:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 504-------------------- +[May 1, 2026 - 08:16:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[May 1, 2026 - 08:16:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[May 1, 2026 - 08:16:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:16:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 08:19:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666849 41.824131 41.535316 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.086426 136.089844 136.076172 +util 65.500000 72.000000 61.000000 +[May 1, 2026 - 08:19:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 505 : avg iteration time 166.27 seconds +[May 1, 2026 - 08:19:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 505 : data loading time 0.00 +[May 1, 2026 - 08:19:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 505 : avg forward pass time 74.54 +[May 1, 2026 - 08:19:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 505 : backward pass time 8.44 +[May 1, 2026 - 08:19:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 505 : optimizer step time 0.40 +[May 1, 2026 - 08:19:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 505-------------------- +[May 1, 2026 - 08:19:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 418580.5625 iter count: 1.0 +[May 1, 2026 - 08:19:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4831 iter count: 1.0 +[May 1, 2026 - 08:19:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 418580.5625iter count: 1.0 +[May 1, 2026 - 08:19:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:19:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 5.0040iter count: 1.0 +[May 1, 2026 - 08:19:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 2.9398 iter count: 1.0 +[May 1, 2026 - 08:19:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.2510 iter count: 1.0 +[May 1, 2026 - 08:19:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 728225.0625iter count: 1.0 +[May 1, 2026 - 08:19:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 357.6684iter count: 1.0 +[May 1, 2026 - 08:19:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 14679587.0000iter count: 1.0 +[May 1, 2026 - 08:19:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4831iter count: 1.0 +[May 1, 2026 - 08:19:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.3427 iter count: 1.0 +[May 1, 2026 - 08:19:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 5915.7485iter count: 1.0 +[May 1, 2026 - 08:19:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.1221 iter count: 1.0 +[May 1, 2026 - 08:19:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 515.0582iter count: 1.0 +[May 1, 2026 - 08:19:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.0471 iter count: 1.0 +[May 1, 2026 - 08:19:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 24204.0332iter count: 1.0 +[May 1, 2026 - 08:19:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.1366 iter count: 1.0 +[May 1, 2026 - 08:19:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 530.1424iter count: 1.0 +[May 1, 2026 - 08:19:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[May 1, 2026 - 08:19:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:19:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 08:20:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666862 41.824116 41.535297 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.505859 67.582031 67.277344 +util 97.250000 100.000000 92.000000 +[May 1, 2026 - 08:20:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 506 : avg iteration time 47.87 seconds +[May 1, 2026 - 08:20:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 506 : data loading time 0.00 +[May 1, 2026 - 08:20:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 506 : avg forward pass time 13.00 +[May 1, 2026 - 08:20:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 506 : backward pass time 10.14 +[May 1, 2026 - 08:20:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 506 : optimizer step time 1.57 +[May 1, 2026 - 08:20:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 506-------------------- +[May 1, 2026 - 08:20:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 08:20:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 08:20:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:20:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 08:20:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666982 41.824615 41.535297 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.393555 67.445312 67.238281 +util 93.250000 100.000000 88.000000 +[May 1, 2026 - 08:20:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 507 : avg iteration time 46.25 seconds +[May 1, 2026 - 08:20:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 507 : data loading time 0.00 +[May 1, 2026 - 08:20:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 507 : avg forward pass time 12.20 +[May 1, 2026 - 08:20:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 507 : backward pass time 10.15 +[May 1, 2026 - 08:20:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 507 : optimizer step time 1.57 +[May 1, 2026 - 08:20:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 507-------------------- +[May 1, 2026 - 08:20:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0101 iter count: 1.0 +[May 1, 2026 - 08:20:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[May 1, 2026 - 08:20:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:20:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 08:21:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666991 41.824623 41.535297 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.403320 67.425781 67.335938 +util 95.750000 99.000000 92.000000 +[May 1, 2026 - 08:21:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 508 : avg iteration time 46.29 seconds +[May 1, 2026 - 08:21:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 508 : data loading time 0.00 +[May 1, 2026 - 08:21:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 508 : avg forward pass time 12.21 +[May 1, 2026 - 08:21:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 508 : backward pass time 10.14 +[May 1, 2026 - 08:21:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 508 : optimizer step time 1.59 +[May 1, 2026 - 08:21:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 508-------------------- +[May 1, 2026 - 08:21:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 08:21:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 08:21:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:21:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 08:22:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.667059 41.824863 41.535320 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.574219 67.960938 67.445312 +util 93.000000 99.000000 88.000000 +[May 1, 2026 - 08:22:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 509 : avg iteration time 48.95 seconds +[May 1, 2026 - 08:22:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 509 : data loading time 0.00 +[May 1, 2026 - 08:22:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 509 : avg forward pass time 13.55 +[May 1, 2026 - 08:22:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 509 : backward pass time 10.15 +[May 1, 2026 - 08:22:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 509 : optimizer step time 1.57 +[May 1, 2026 - 08:22:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 509-------------------- +[May 1, 2026 - 08:22:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 08:22:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 08:22:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:22:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 08:25:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666856 41.824032 41.535332 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.018066 136.154297 135.972656 +util 63.500000 70.000000 58.000000 +[May 1, 2026 - 08:25:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 510 : avg iteration time 163.22 seconds +[May 1, 2026 - 08:25:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 510 : data loading time 0.00 +[May 1, 2026 - 08:25:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 510 : avg forward pass time 72.99 +[May 1, 2026 - 08:25:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 510 : backward pass time 8.45 +[May 1, 2026 - 08:25:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 510 : optimizer step time 0.40 +[May 1, 2026 - 08:25:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 510-------------------- +[May 1, 2026 - 08:25:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 1032891.1565 iter count: 1.0 +[May 1, 2026 - 08:25:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5186 iter count: 1.0 +[May 1, 2026 - 08:25:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 1032891.1565iter count: 1.0 +[May 1, 2026 - 08:25:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:25:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.6715iter count: 1.0 +[May 1, 2026 - 08:25:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.9936 iter count: 1.0 +[May 1, 2026 - 08:25:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.9679 iter count: 1.0 +[May 1, 2026 - 08:25:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 2163638.2500iter count: 1.0 +[May 1, 2026 - 08:25:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 7.2952 iter count: 1.0 +[May 1, 2026 - 08:25:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 61573456.0000iter count: 1.0 +[May 1, 2026 - 08:25:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5186iter count: 1.0 +[May 1, 2026 - 08:25:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.2603 iter count: 1.0 +[May 1, 2026 - 08:25:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 5016.9546iter count: 1.0 +[May 1, 2026 - 08:25:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.9665 iter count: 1.0 +[May 1, 2026 - 08:25:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 152216.1562iter count: 1.0 +[May 1, 2026 - 08:25:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.1117 iter count: 1.0 +[May 1, 2026 - 08:25:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 27539.0176iter count: 1.0 +[May 1, 2026 - 08:25:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.6380 iter count: 1.0 +[May 1, 2026 - 08:25:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 1445.0731iter count: 1.0 +[May 1, 2026 - 08:25:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 1, 2026 - 08:25:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:25:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 08:26:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666845 41.824020 41.535324 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.637695 67.921875 67.542969 +util 93.000000 99.000000 88.000000 +[May 1, 2026 - 08:26:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 511 : avg iteration time 50.38 seconds +[May 1, 2026 - 08:26:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 511 : data loading time 0.00 +[May 1, 2026 - 08:26:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 511 : avg forward pass time 14.28 +[May 1, 2026 - 08:26:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 511 : backward pass time 10.11 +[May 1, 2026 - 08:26:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 511 : optimizer step time 1.56 +[May 1, 2026 - 08:26:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 511-------------------- +[May 1, 2026 - 08:26:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0112 iter count: 1.0 +[May 1, 2026 - 08:26:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 1, 2026 - 08:26:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:26:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 08:26:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666910 41.824272 41.535301 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.535156 67.921875 67.406250 +util 95.750000 99.000000 90.000000 +[May 1, 2026 - 08:26:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 512 : avg iteration time 46.29 seconds +[May 1, 2026 - 08:26:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 512 : data loading time 0.00 +[May 1, 2026 - 08:26:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 512 : avg forward pass time 12.23 +[May 1, 2026 - 08:26:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 512 : backward pass time 10.14 +[May 1, 2026 - 08:26:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 512 : optimizer step time 1.56 +[May 1, 2026 - 08:26:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 512-------------------- +[May 1, 2026 - 08:26:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 08:26:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 08:26:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:26:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 08:27:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.667053 41.824780 41.535343 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.662109 67.960938 67.562500 +util 95.250000 99.000000 91.000000 +[May 1, 2026 - 08:27:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 513 : avg iteration time 43.50 seconds +[May 1, 2026 - 08:27:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 513 : data loading time 0.00 +[May 1, 2026 - 08:27:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 513 : avg forward pass time 10.83 +[May 1, 2026 - 08:27:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 513 : backward pass time 10.14 +[May 1, 2026 - 08:27:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 513 : optimizer step time 1.57 +[May 1, 2026 - 08:27:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 513-------------------- +[May 1, 2026 - 08:27:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0114 iter count: 1.0 +[May 1, 2026 - 08:27:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 1, 2026 - 08:27:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:27:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 08:28:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.667060 41.824780 41.535343 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.627930 68.000000 67.503906 +util 93.750000 100.000000 89.000000 +[May 1, 2026 - 08:28:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 514 : avg iteration time 43.57 seconds +[May 1, 2026 - 08:28:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 514 : data loading time 0.00 +[May 1, 2026 - 08:28:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 514 : avg forward pass time 10.86 +[May 1, 2026 - 08:28:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 514 : backward pass time 10.14 +[May 1, 2026 - 08:28:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 514 : optimizer step time 1.56 +[May 1, 2026 - 08:28:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 514-------------------- +[May 1, 2026 - 08:28:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0112 iter count: 1.0 +[May 1, 2026 - 08:28:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 1, 2026 - 08:28:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:28:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 08:31:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666844 41.823910 41.535320 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.032715 136.037109 136.031250 +util 62.750000 69.000000 58.000000 +[May 1, 2026 - 08:31:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 515 : avg iteration time 166.06 seconds +[May 1, 2026 - 08:31:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 515 : data loading time 0.00 +[May 1, 2026 - 08:31:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 515 : avg forward pass time 74.44 +[May 1, 2026 - 08:31:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 515 : backward pass time 8.41 +[May 1, 2026 - 08:31:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 515 : optimizer step time 0.40 +[May 1, 2026 - 08:31:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 515-------------------- +[May 1, 2026 - 08:31:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 240873.5781 iter count: 1.0 +[May 1, 2026 - 08:31:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5905 iter count: 1.0 +[May 1, 2026 - 08:31:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 240873.5781iter count: 1.0 +[May 1, 2026 - 08:31:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:31:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.4252iter count: 1.0 +[May 1, 2026 - 08:31:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.9882 iter count: 1.0 +[May 1, 2026 - 08:31:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.6911 iter count: 1.0 +[May 1, 2026 - 08:31:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 355020.0625iter count: 1.0 +[May 1, 2026 - 08:31:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 7.2170 iter count: 1.0 +[May 1, 2026 - 08:31:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 4790996.0000iter count: 1.0 +[May 1, 2026 - 08:31:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5905iter count: 1.0 +[May 1, 2026 - 08:31:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 7.6911 iter count: 1.0 +[May 1, 2026 - 08:31:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 4790996.0000iter count: 1.0 +[May 1, 2026 - 08:31:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 6.4479 iter count: 1.0 +[May 1, 2026 - 08:31:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 398639.2188iter count: 1.0 +[May 1, 2026 - 08:31:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.4395 iter count: 1.0 +[May 1, 2026 - 08:31:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 7180.2085iter count: 1.0 +[May 1, 2026 - 08:31:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.9741 iter count: 1.0 +[May 1, 2026 - 08:31:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 2830.4111iter count: 1.0 +[May 1, 2026 - 08:31:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 08:31:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:31:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 08:31:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666846 41.823921 41.535339 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.505859 67.562500 67.335938 +util 93.000000 99.000000 88.000000 +[May 1, 2026 - 08:31:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 516 : avg iteration time 50.53 seconds +[May 1, 2026 - 08:31:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 516 : data loading time 0.00 +[May 1, 2026 - 08:31:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 516 : avg forward pass time 14.34 +[May 1, 2026 - 08:31:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 516 : backward pass time 10.13 +[May 1, 2026 - 08:31:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 516 : optimizer step time 1.58 +[May 1, 2026 - 08:31:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 516-------------------- +[May 1, 2026 - 08:31:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0119 iter count: 1.0 +[May 1, 2026 - 08:31:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[May 1, 2026 - 08:31:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:31:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 08:32:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.667009 41.824417 41.535336 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.769531 67.953125 67.218750 +util 96.250000 100.000000 91.000000 +[May 1, 2026 - 08:32:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 517 : avg iteration time 43.54 seconds +[May 1, 2026 - 08:32:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 517 : data loading time 0.00 +[May 1, 2026 - 08:32:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 517 : avg forward pass time 10.85 +[May 1, 2026 - 08:32:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 517 : backward pass time 10.14 +[May 1, 2026 - 08:32:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 517 : optimizer step time 1.57 +[May 1, 2026 - 08:32:40 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 517-------------------- +[May 1, 2026 - 08:32:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0117 iter count: 1.0 +[May 1, 2026 - 08:32:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[May 1, 2026 - 08:32:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:32:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 08:33:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.667002 41.824425 41.535336 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.344727 67.406250 67.160156 +util 93.750000 99.000000 89.000000 +[May 1, 2026 - 08:33:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 518 : avg iteration time 43.68 seconds +[May 1, 2026 - 08:33:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 518 : data loading time 0.00 +[May 1, 2026 - 08:33:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 518 : avg forward pass time 10.91 +[May 1, 2026 - 08:33:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 518 : backward pass time 10.14 +[May 1, 2026 - 08:33:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 518 : optimizer step time 1.58 +[May 1, 2026 - 08:33:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 518-------------------- +[May 1, 2026 - 08:33:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0117 iter count: 1.0 +[May 1, 2026 - 08:33:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[May 1, 2026 - 08:33:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:33:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 08:34:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666999 41.824436 41.535336 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.520508 67.582031 67.335938 +util 93.000000 99.000000 88.000000 +[May 1, 2026 - 08:34:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 519 : avg iteration time 46.19 seconds +[May 1, 2026 - 08:34:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 519 : data loading time 0.00 +[May 1, 2026 - 08:34:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 519 : avg forward pass time 12.16 +[May 1, 2026 - 08:34:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 519 : backward pass time 10.13 +[May 1, 2026 - 08:34:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 519 : optimizer step time 1.57 +[May 1, 2026 - 08:34:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 519-------------------- +[May 1, 2026 - 08:34:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 08:34:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 08:34:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:34:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 08:37:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666862 41.823818 41.535366 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.026367 136.029297 136.017578 +util 66.000000 73.000000 60.000000 +[May 1, 2026 - 08:37:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 520 : avg iteration time 171.04 seconds +[May 1, 2026 - 08:37:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 520 : data loading time 0.00 +[May 1, 2026 - 08:37:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 520 : avg forward pass time 76.91 +[May 1, 2026 - 08:37:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 520 : backward pass time 8.44 +[May 1, 2026 - 08:37:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 520 : optimizer step time 0.39 +[May 1, 2026 - 08:37:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 520-------------------- +[May 1, 2026 - 08:37:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 2320603.1914 iter count: 1.0 +[May 1, 2026 - 08:37:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5313 iter count: 1.0 +[May 1, 2026 - 08:37:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 2320603.1914iter count: 1.0 +[May 1, 2026 - 08:37:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:37:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.8116iter count: 1.0 +[May 1, 2026 - 08:37:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.7682 iter count: 1.0 +[May 1, 2026 - 08:37:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 9.0955 iter count: 1.0 +[May 1, 2026 - 08:37:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 3293640.0000iter count: 1.0 +[May 1, 2026 - 08:37:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 4.6474 iter count: 1.0 +[May 1, 2026 - 08:37:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 79484640.0000iter count: 1.0 +[May 1, 2026 - 08:37:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5313iter count: 1.0 +[May 1, 2026 - 08:37:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 1.4246 iter count: 1.0 +[May 1, 2026 - 08:37:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 17.2741 iter count: 1.0 +[May 1, 2026 - 08:37:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.3826 iter count: 1.0 +[May 1, 2026 - 08:37:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 6407.6221iter count: 1.0 +[May 1, 2026 - 08:37:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.9129 iter count: 1.0 +[May 1, 2026 - 08:37:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 18505.1387iter count: 1.0 +[May 1, 2026 - 08:37:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.9422 iter count: 1.0 +[May 1, 2026 - 08:37:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 1071332.5000iter count: 1.0 +[May 1, 2026 - 08:37:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 08:37:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:37:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 08:37:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666871 41.823822 41.535366 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.554199 67.939453 67.425781 +util 95.250000 99.000000 89.000000 +[May 1, 2026 - 08:37:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 521 : avg iteration time 47.88 seconds +[May 1, 2026 - 08:37:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 521 : data loading time 0.00 +[May 1, 2026 - 08:37:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 521 : avg forward pass time 13.04 +[May 1, 2026 - 08:37:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 521 : backward pass time 10.12 +[May 1, 2026 - 08:37:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 521 : optimizer step time 1.56 +[May 1, 2026 - 08:37:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 521-------------------- +[May 1, 2026 - 08:37:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0117 iter count: 1.0 +[May 1, 2026 - 08:37:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[May 1, 2026 - 08:37:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:37:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 08:38:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.667007 41.824387 41.535347 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.441895 67.958984 67.269531 +util 92.250000 99.000000 87.000000 +[May 1, 2026 - 08:38:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 522 : avg iteration time 48.87 seconds +[May 1, 2026 - 08:38:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 522 : data loading time 0.00 +[May 1, 2026 - 08:38:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 522 : avg forward pass time 13.52 +[May 1, 2026 - 08:38:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 522 : backward pass time 10.13 +[May 1, 2026 - 08:38:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 522 : optimizer step time 1.58 +[May 1, 2026 - 08:38:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 522-------------------- +[May 1, 2026 - 08:38:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 08:38:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[May 1, 2026 - 08:38:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:38:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 08:39:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666827 41.823681 41.535370 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.417480 67.978516 67.230469 +util 96.000000 99.000000 90.000000 +[May 1, 2026 - 08:39:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 523 : avg iteration time 43.55 seconds +[May 1, 2026 - 08:39:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 523 : data loading time 0.00 +[May 1, 2026 - 08:39:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 523 : avg forward pass time 10.85 +[May 1, 2026 - 08:39:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 523 : backward pass time 10.12 +[May 1, 2026 - 08:39:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 523 : optimizer step time 1.58 +[May 1, 2026 - 08:39:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 523-------------------- +[May 1, 2026 - 08:39:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0119 iter count: 1.0 +[May 1, 2026 - 08:39:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[May 1, 2026 - 08:39:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:39:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 08:40:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666779 41.823437 41.535362 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.524902 67.763672 67.445312 +util 93.750000 99.000000 89.000000 +[May 1, 2026 - 08:40:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 524 : avg iteration time 48.92 seconds +[May 1, 2026 - 08:40:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 524 : data loading time 0.00 +[May 1, 2026 - 08:40:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 524 : avg forward pass time 13.55 +[May 1, 2026 - 08:40:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 524 : backward pass time 10.13 +[May 1, 2026 - 08:40:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 524 : optimizer step time 1.56 +[May 1, 2026 - 08:40:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 524-------------------- +[May 1, 2026 - 08:40:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 08:40:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[May 1, 2026 - 08:40:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:40:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 08:42:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666610 41.822777 41.535366 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.099609 136.107422 136.076172 +util 63.250000 70.000000 58.000000 +[May 1, 2026 - 08:42:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 525 : avg iteration time 163.19 seconds +[May 1, 2026 - 08:42:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 525 : data loading time 0.00 +[May 1, 2026 - 08:42:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 525 : avg forward pass time 72.97 +[May 1, 2026 - 08:42:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 525 : backward pass time 8.41 +[May 1, 2026 - 08:42:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 525 : optimizer step time 0.39 +[May 1, 2026 - 08:42:54 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 525-------------------- +[May 1, 2026 - 08:42:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 41156.8649 iter count: 1.0 +[May 1, 2026 - 08:42:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4973 iter count: 1.0 +[May 1, 2026 - 08:42:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 41156.8649iter count: 1.0 +[May 1, 2026 - 08:42:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:42:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.1300iter count: 1.0 +[May 1, 2026 - 08:42:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.8101 iter count: 1.0 +[May 1, 2026 - 08:42:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 6.9663 iter count: 1.0 +[May 1, 2026 - 08:42:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 99397.5469iter count: 1.0 +[May 1, 2026 - 08:42:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 5.0541 iter count: 1.0 +[May 1, 2026 - 08:42:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 1124145.0000iter count: 1.0 +[May 1, 2026 - 08:42:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4973iter count: 1.0 +[May 1, 2026 - 08:42:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 1.0334 iter count: 1.0 +[May 1, 2026 - 08:42:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 7.8992 iter count: 1.0 +[May 1, 2026 - 08:42:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.8848 iter count: 1.0 +[May 1, 2026 - 08:42:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 129262.5547iter count: 1.0 +[May 1, 2026 - 08:42:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 6.6202 iter count: 1.0 +[May 1, 2026 - 08:42:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 562671.3125iter count: 1.0 +[May 1, 2026 - 08:42:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.7636 iter count: 1.0 +[May 1, 2026 - 08:42:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 1858.0178iter count: 1.0 +[May 1, 2026 - 08:42:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 1, 2026 - 08:42:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:42:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 08:43:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666637 41.822868 41.535381 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.534668 67.919922 67.406250 +util 98.250000 100.000000 94.000000 +[May 1, 2026 - 08:43:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 526 : avg iteration time 45.12 seconds +[May 1, 2026 - 08:43:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 526 : data loading time 0.00 +[May 1, 2026 - 08:43:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 526 : avg forward pass time 11.64 +[May 1, 2026 - 08:43:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 526 : backward pass time 10.12 +[May 1, 2026 - 08:43:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 526 : optimizer step time 1.57 +[May 1, 2026 - 08:43:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 526-------------------- +[May 1, 2026 - 08:43:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 1, 2026 - 08:43:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 1, 2026 - 08:43:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:43:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 08:44:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.667004 41.824352 41.535381 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.554199 67.998047 67.406250 +util 97.000000 100.000000 92.000000 +[May 1, 2026 - 08:44:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 527 : avg iteration time 48.97 seconds +[May 1, 2026 - 08:44:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 527 : data loading time 0.00 +[May 1, 2026 - 08:44:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 527 : avg forward pass time 13.55 +[May 1, 2026 - 08:44:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 527 : backward pass time 10.14 +[May 1, 2026 - 08:44:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 527 : optimizer step time 1.57 +[May 1, 2026 - 08:44:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 527-------------------- +[May 1, 2026 - 08:44:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0114 iter count: 1.0 +[May 1, 2026 - 08:44:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 1, 2026 - 08:44:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:44:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 08:45:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.667005 41.824329 41.535400 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.510254 67.763672 67.425781 +util 96.000000 99.000000 92.000000 +[May 1, 2026 - 08:45:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 528 : avg iteration time 43.54 seconds +[May 1, 2026 - 08:45:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 528 : data loading time 0.00 +[May 1, 2026 - 08:45:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 528 : avg forward pass time 10.83 +[May 1, 2026 - 08:45:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 528 : backward pass time 10.14 +[May 1, 2026 - 08:45:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 528 : optimizer step time 1.58 +[May 1, 2026 - 08:45:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 528-------------------- +[May 1, 2026 - 08:45:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0119 iter count: 1.0 +[May 1, 2026 - 08:45:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[May 1, 2026 - 08:45:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:45:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 08:46:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.667000 41.824329 41.535381 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.529785 67.900391 67.406250 +util 96.000000 99.000000 92.000000 +[May 1, 2026 - 08:46:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 529 : avg iteration time 49.00 seconds +[May 1, 2026 - 08:46:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 529 : data loading time 0.00 +[May 1, 2026 - 08:46:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 529 : avg forward pass time 13.57 +[May 1, 2026 - 08:46:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 529 : backward pass time 10.14 +[May 1, 2026 - 08:46:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 529 : optimizer step time 1.59 +[May 1, 2026 - 08:46:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 529-------------------- +[May 1, 2026 - 08:46:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 08:46:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 08:46:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:46:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 08:48:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666698 41.823120 41.535381 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 135.974121 136.037109 135.953125 +util 65.250000 73.000000 59.000000 +[May 1, 2026 - 08:48:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 530 : avg iteration time 163.14 seconds +[May 1, 2026 - 08:48:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 530 : data loading time 0.00 +[May 1, 2026 - 08:48:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 530 : avg forward pass time 72.96 +[May 1, 2026 - 08:48:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 530 : backward pass time 8.42 +[May 1, 2026 - 08:48:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 530 : optimizer step time 0.39 +[May 1, 2026 - 08:48:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 530-------------------- +[May 1, 2026 - 08:48:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 212786.5083 iter count: 1.0 +[May 1, 2026 - 08:48:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5840 iter count: 1.0 +[May 1, 2026 - 08:48:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 212786.5083iter count: 1.0 +[May 1, 2026 - 08:48:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:48:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.7450iter count: 1.0 +[May 1, 2026 - 08:48:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 2.3827 iter count: 1.0 +[May 1, 2026 - 08:48:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.9498 iter count: 1.0 +[May 1, 2026 - 08:48:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 394934.2812iter count: 1.0 +[May 1, 2026 - 08:48:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 117.3837iter count: 1.0 +[May 1, 2026 - 08:48:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 8036817.5000iter count: 1.0 +[May 1, 2026 - 08:48:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5840iter count: 1.0 +[May 1, 2026 - 08:48:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.0218 iter count: 1.0 +[May 1, 2026 - 08:48:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 421.3712iter count: 1.0 +[May 1, 2026 - 08:48:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.4291 iter count: 1.0 +[May 1, 2026 - 08:48:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 51954.7070iter count: 1.0 +[May 1, 2026 - 08:48:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.1581 iter count: 1.0 +[May 1, 2026 - 08:48:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 30219.7148iter count: 1.0 +[May 1, 2026 - 08:48:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.4689 iter count: 1.0 +[May 1, 2026 - 08:48:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 56263.3984iter count: 1.0 +[May 1, 2026 - 08:48:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[May 1, 2026 - 08:48:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:48:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 08:49:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666701 41.823120 41.535397 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.144043 67.171875 67.060547 +util 93.000000 99.000000 88.000000 +[May 1, 2026 - 08:49:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 531 : avg iteration time 50.56 seconds +[May 1, 2026 - 08:49:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 531 : data loading time 0.00 +[May 1, 2026 - 08:49:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 531 : avg forward pass time 14.36 +[May 1, 2026 - 08:49:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 531 : backward pass time 10.14 +[May 1, 2026 - 08:49:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 531 : optimizer step time 1.56 +[May 1, 2026 - 08:49:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 531-------------------- +[May 1, 2026 - 08:49:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0140 iter count: 1.0 +[May 1, 2026 - 08:49:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0140 iter count: 1.0 +[May 1, 2026 - 08:49:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:49:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 08:50:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666817 41.823570 41.535408 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.305176 67.308594 67.294922 +util 95.500000 100.000000 91.000000 +[May 1, 2026 - 08:50:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 532 : avg iteration time 48.88 seconds +[May 1, 2026 - 08:50:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 532 : data loading time 0.00 +[May 1, 2026 - 08:50:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 532 : avg forward pass time 13.51 +[May 1, 2026 - 08:50:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 532 : backward pass time 10.13 +[May 1, 2026 - 08:50:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 532 : optimizer step time 1.57 +[May 1, 2026 - 08:50:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 532-------------------- +[May 1, 2026 - 08:50:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0111 iter count: 1.0 +[May 1, 2026 - 08:50:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 1, 2026 - 08:50:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:50:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 08:51:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666816 41.823578 41.535397 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.588379 67.679688 67.314453 +util 94.000000 99.000000 89.000000 +[May 1, 2026 - 08:51:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 533 : avg iteration time 46.18 seconds +[May 1, 2026 - 08:51:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 533 : data loading time 0.00 +[May 1, 2026 - 08:51:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 533 : avg forward pass time 12.17 +[May 1, 2026 - 08:51:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 533 : backward pass time 10.12 +[May 1, 2026 - 08:51:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 533 : optimizer step time 1.57 +[May 1, 2026 - 08:51:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 533-------------------- +[May 1, 2026 - 08:51:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 08:51:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 08:51:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:51:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 08:51:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666900 41.823856 41.535397 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.212402 67.230469 67.158203 +util 96.500000 99.000000 91.000000 +[May 1, 2026 - 08:51:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 534 : avg iteration time 46.29 seconds +[May 1, 2026 - 08:51:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 534 : data loading time 0.00 +[May 1, 2026 - 08:51:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 534 : avg forward pass time 12.22 +[May 1, 2026 - 08:51:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 534 : backward pass time 10.13 +[May 1, 2026 - 08:51:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 534 : optimizer step time 1.57 +[May 1, 2026 - 08:51:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 534-------------------- +[May 1, 2026 - 08:51:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 08:51:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 08:51:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:51:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 08:54:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666745 41.823215 41.535419 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.022949 136.031250 135.998047 +util 64.000000 72.000000 58.000000 +[May 1, 2026 - 08:54:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 535 : avg iteration time 163.27 seconds +[May 1, 2026 - 08:54:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 535 : data loading time 0.00 +[May 1, 2026 - 08:54:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 535 : avg forward pass time 73.03 +[May 1, 2026 - 08:54:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 535 : backward pass time 8.43 +[May 1, 2026 - 08:54:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 535 : optimizer step time 0.40 +[May 1, 2026 - 08:54:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 535-------------------- +[May 1, 2026 - 08:54:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 1106925.2041 iter count: 1.0 +[May 1, 2026 - 08:54:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5150 iter count: 1.0 +[May 1, 2026 - 08:54:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 1106925.2041iter count: 1.0 +[May 1, 2026 - 08:54:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:54:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.2570iter count: 1.0 +[May 1, 2026 - 08:54:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.8675 iter count: 1.0 +[May 1, 2026 - 08:54:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.8710 iter count: 1.0 +[May 1, 2026 - 08:54:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 1863134.5000iter count: 1.0 +[May 1, 2026 - 08:54:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 5.6684 iter count: 1.0 +[May 1, 2026 - 08:54:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 50729672.0000iter count: 1.0 +[May 1, 2026 - 08:54:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5150iter count: 1.0 +[May 1, 2026 - 08:54:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 7.5825 iter count: 1.0 +[May 1, 2026 - 08:54:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 3855614.7500iter count: 1.0 +[May 1, 2026 - 08:54:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 2.3472 iter count: 1.0 +[May 1, 2026 - 08:54:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 109.3412iter count: 1.0 +[May 1, 2026 - 08:54:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.3701 iter count: 1.0 +[May 1, 2026 - 08:54:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 6248.7290iter count: 1.0 +[May 1, 2026 - 08:54:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 7.5515 iter count: 1.0 +[May 1, 2026 - 08:54:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 3623434.5000iter count: 1.0 +[May 1, 2026 - 08:54:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[May 1, 2026 - 08:54:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:54:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 08:55:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666759 41.823212 41.535450 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.793457 67.933594 67.373047 +util 95.250000 100.000000 91.000000 +[May 1, 2026 - 08:55:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 536 : avg iteration time 47.81 seconds +[May 1, 2026 - 08:55:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 536 : data loading time 0.00 +[May 1, 2026 - 08:55:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 536 : avg forward pass time 12.97 +[May 1, 2026 - 08:55:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 536 : backward pass time 10.16 +[May 1, 2026 - 08:55:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 536 : optimizer step time 1.57 +[May 1, 2026 - 08:55:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 536-------------------- +[May 1, 2026 - 08:55:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0123 iter count: 1.0 +[May 1, 2026 - 08:55:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0123 iter count: 1.0 +[May 1, 2026 - 08:55:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:55:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 08:56:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.667013 41.824188 41.535473 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.852051 67.953125 67.548828 +util 94.500000 100.000000 90.000000 +[May 1, 2026 - 08:56:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 537 : avg iteration time 48.99 seconds +[May 1, 2026 - 08:56:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 537 : data loading time 0.00 +[May 1, 2026 - 08:56:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 537 : avg forward pass time 13.56 +[May 1, 2026 - 08:56:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 537 : backward pass time 10.15 +[May 1, 2026 - 08:56:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 537 : optimizer step time 1.56 +[May 1, 2026 - 08:56:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 537-------------------- +[May 1, 2026 - 08:56:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0108 iter count: 1.0 +[May 1, 2026 - 08:56:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 1, 2026 - 08:56:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:56:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 08:57:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.667014 41.824192 41.535473 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.944824 67.978516 67.933594 +util 94.500000 98.000000 88.000000 +[May 1, 2026 - 08:57:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 538 : avg iteration time 46.29 seconds +[May 1, 2026 - 08:57:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 538 : data loading time 0.00 +[May 1, 2026 - 08:57:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 538 : avg forward pass time 12.23 +[May 1, 2026 - 08:57:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 538 : backward pass time 10.13 +[May 1, 2026 - 08:57:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 538 : optimizer step time 1.57 +[May 1, 2026 - 08:57:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 538-------------------- +[May 1, 2026 - 08:57:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 08:57:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 08:57:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:57:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 08:57:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.667009 41.824196 41.535427 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.944824 67.972656 67.861328 +util 93.000000 99.000000 88.000000 +[May 1, 2026 - 08:57:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 539 : avg iteration time 48.94 seconds +[May 1, 2026 - 08:57:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 539 : data loading time 0.00 +[May 1, 2026 - 08:57:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 539 : avg forward pass time 13.55 +[May 1, 2026 - 08:57:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 539 : backward pass time 10.13 +[May 1, 2026 - 08:57:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 539 : optimizer step time 1.58 +[May 1, 2026 - 08:57:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 539-------------------- +[May 1, 2026 - 08:57:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 08:57:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[May 1, 2026 - 08:57:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 08:57:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 09:00:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666673 41.822865 41.535435 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.022949 136.056641 136.011719 +util 64.250000 70.000000 60.000000 +[May 1, 2026 - 09:00:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 540 : avg iteration time 166.02 seconds +[May 1, 2026 - 09:00:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 540 : data loading time 0.00 +[May 1, 2026 - 09:00:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 540 : avg forward pass time 74.41 +[May 1, 2026 - 09:00:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 540 : backward pass time 8.43 +[May 1, 2026 - 09:00:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 540 : optimizer step time 0.39 +[May 1, 2026 - 09:00:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 540-------------------- +[May 1, 2026 - 09:00:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 171029.1968 iter count: 1.0 +[May 1, 2026 - 09:00:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5799 iter count: 1.0 +[May 1, 2026 - 09:00:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 171029.1968iter count: 1.0 +[May 1, 2026 - 09:00:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:00:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.6381iter count: 1.0 +[May 1, 2026 - 09:00:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 2.2606 iter count: 1.0 +[May 1, 2026 - 09:00:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.5894 iter count: 1.0 +[May 1, 2026 - 09:00:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 300968.5000iter count: 1.0 +[May 1, 2026 - 09:00:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 91.9420iter count: 1.0 +[May 1, 2026 - 09:00:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 3908999.2500iter count: 1.0 +[May 1, 2026 - 09:00:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5799iter count: 1.0 +[May 1, 2026 - 09:00:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.5098 iter count: 1.0 +[May 1, 2026 - 09:00:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 61065.1875iter count: 1.0 +[May 1, 2026 - 09:00:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 6.1761 iter count: 1.0 +[May 1, 2026 - 09:00:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 231455.8125iter count: 1.0 +[May 1, 2026 - 09:00:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.8375 iter count: 1.0 +[May 1, 2026 - 09:00:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 2153.6824iter count: 1.0 +[May 1, 2026 - 09:00:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.6995 iter count: 1.0 +[May 1, 2026 - 09:00:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 12075.7021iter count: 1.0 +[May 1, 2026 - 09:00:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 1, 2026 - 09:00:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:00:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 09:01:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666676 41.822853 41.535480 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.417480 67.445312 67.333984 +util 92.250000 99.000000 87.000000 +[May 1, 2026 - 09:01:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 541 : avg iteration time 50.57 seconds +[May 1, 2026 - 09:01:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 541 : data loading time 0.00 +[May 1, 2026 - 09:01:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 541 : avg forward pass time 14.35 +[May 1, 2026 - 09:01:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 541 : backward pass time 10.15 +[May 1, 2026 - 09:01:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 541 : optimizer step time 1.57 +[May 1, 2026 - 09:01:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 541-------------------- +[May 1, 2026 - 09:01:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0107 iter count: 1.0 +[May 1, 2026 - 09:01:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[May 1, 2026 - 09:01:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:01:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 09:02:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.667051 41.824379 41.535435 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.305176 67.367188 67.119141 +util 94.250000 99.000000 90.000000 +[May 1, 2026 - 09:02:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 542 : avg iteration time 46.23 seconds +[May 1, 2026 - 09:02:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 542 : data loading time 0.00 +[May 1, 2026 - 09:02:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 542 : avg forward pass time 12.19 +[May 1, 2026 - 09:02:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 542 : backward pass time 10.12 +[May 1, 2026 - 09:02:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 542 : optimizer step time 1.55 +[May 1, 2026 - 09:02:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 542-------------------- +[May 1, 2026 - 09:02:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 09:02:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 09:02:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:02:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 09:03:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.667046 41.824360 41.535454 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.388184 67.425781 67.275391 +util 97.750000 100.000000 93.000000 +[May 1, 2026 - 09:03:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 543 : avg iteration time 46.30 seconds +[May 1, 2026 - 09:03:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 543 : data loading time 0.00 +[May 1, 2026 - 09:03:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 543 : avg forward pass time 12.23 +[May 1, 2026 - 09:03:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 543 : backward pass time 10.14 +[May 1, 2026 - 09:03:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 543 : optimizer step time 1.56 +[May 1, 2026 - 09:03:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 543-------------------- +[May 1, 2026 - 09:03:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0116 iter count: 1.0 +[May 1, 2026 - 09:03:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[May 1, 2026 - 09:03:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:03:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 09:03:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.667049 41.824360 41.535442 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.344238 67.367188 67.275391 +util 94.500000 98.000000 88.000000 +[May 1, 2026 - 09:03:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 544 : avg iteration time 48.94 seconds +[May 1, 2026 - 09:03:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 544 : data loading time 0.00 +[May 1, 2026 - 09:03:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 544 : avg forward pass time 13.52 +[May 1, 2026 - 09:03:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 544 : backward pass time 10.16 +[May 1, 2026 - 09:03:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 544 : optimizer step time 1.58 +[May 1, 2026 - 09:03:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 544-------------------- +[May 1, 2026 - 09:03:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 1, 2026 - 09:03:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 1, 2026 - 09:03:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:03:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 09:06:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666618 41.822613 41.535465 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.045898 136.048828 136.037109 +util 66.500000 73.000000 62.000000 +[May 1, 2026 - 09:06:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 545 : avg iteration time 165.80 seconds +[May 1, 2026 - 09:06:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 545 : data loading time 0.00 +[May 1, 2026 - 09:06:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 545 : avg forward pass time 74.29 +[May 1, 2026 - 09:06:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 545 : backward pass time 8.44 +[May 1, 2026 - 09:06:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 545 : optimizer step time 0.40 +[May 1, 2026 - 09:06:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 545-------------------- +[May 1, 2026 - 09:06:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 157528.4336 iter count: 1.0 +[May 1, 2026 - 09:06:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5937 iter count: 1.0 +[May 1, 2026 - 09:06:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 157528.4336iter count: 1.0 +[May 1, 2026 - 09:06:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:06:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.5912iter count: 1.0 +[May 1, 2026 - 09:06:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.4435 iter count: 1.0 +[May 1, 2026 - 09:06:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.4367 iter count: 1.0 +[May 1, 2026 - 09:06:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 280888.3438iter count: 1.0 +[May 1, 2026 - 09:06:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 17.9406iter count: 1.0 +[May 1, 2026 - 09:06:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 2880499.0000iter count: 1.0 +[May 1, 2026 - 09:06:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5937iter count: 1.0 +[May 1, 2026 - 09:06:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 6.1469 iter count: 1.0 +[May 1, 2026 - 09:06:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 218318.1719iter count: 1.0 +[May 1, 2026 - 09:06:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.5029 iter count: 1.0 +[May 1, 2026 - 09:06:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 60218.1836iter count: 1.0 +[May 1, 2026 - 09:06:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 2.5004 iter count: 1.0 +[May 1, 2026 - 09:06:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 148.5301iter count: 1.0 +[May 1, 2026 - 09:06:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.8960 iter count: 1.0 +[May 1, 2026 - 09:06:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 976854.7500iter count: 1.0 +[May 1, 2026 - 09:06:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[May 1, 2026 - 09:06:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:06:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.23 seconds +[May 1, 2026 - 09:07:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666615 41.822613 41.535450 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.686035 67.816406 67.294922 +util 93.500000 99.000000 89.000000 +[May 1, 2026 - 09:07:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 546 : avg iteration time 47.95 seconds +[May 1, 2026 - 09:07:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 546 : data loading time 0.00 +[May 1, 2026 - 09:07:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 546 : avg forward pass time 12.96 +[May 1, 2026 - 09:07:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 546 : backward pass time 10.13 +[May 1, 2026 - 09:07:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 546 : optimizer step time 1.55 +[May 1, 2026 - 09:07:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 546-------------------- +[May 1, 2026 - 09:07:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0116 iter count: 1.0 +[May 1, 2026 - 09:07:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[May 1, 2026 - 09:07:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:07:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 09:08:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666934 41.823872 41.535450 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.778809 67.816406 67.666016 +util 92.500000 99.000000 88.000000 +[May 1, 2026 - 09:08:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 547 : avg iteration time 43.59 seconds +[May 1, 2026 - 09:08:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 547 : data loading time 0.00 +[May 1, 2026 - 09:08:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 547 : avg forward pass time 10.87 +[May 1, 2026 - 09:08:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 547 : backward pass time 10.15 +[May 1, 2026 - 09:08:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 547 : optimizer step time 1.57 +[May 1, 2026 - 09:08:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 547-------------------- +[May 1, 2026 - 09:08:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0114 iter count: 1.0 +[May 1, 2026 - 09:08:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 1, 2026 - 09:08:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:08:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 09:08:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666933 41.823864 41.535450 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.856934 68.050781 67.275391 +util 94.000000 99.000000 89.000000 +[May 1, 2026 - 09:08:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 548 : avg iteration time 48.97 seconds +[May 1, 2026 - 09:08:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 548 : data loading time 0.00 +[May 1, 2026 - 09:08:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 548 : avg forward pass time 13.56 +[May 1, 2026 - 09:08:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 548 : backward pass time 10.15 +[May 1, 2026 - 09:08:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 548 : optimizer step time 1.56 +[May 1, 2026 - 09:08:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 548-------------------- +[May 1, 2026 - 09:08:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 09:08:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 09:08:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:08:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 09:09:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666940 41.823895 41.535450 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.798340 67.802734 67.796875 +util 95.000000 100.000000 91.000000 +[May 1, 2026 - 09:09:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 549 : avg iteration time 49.01 seconds +[May 1, 2026 - 09:09:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 549 : data loading time 0.00 +[May 1, 2026 - 09:09:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 549 : avg forward pass time 13.59 +[May 1, 2026 - 09:09:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 549 : backward pass time 10.15 +[May 1, 2026 - 09:09:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 549 : optimizer step time 1.57 +[May 1, 2026 - 09:09:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 549-------------------- +[May 1, 2026 - 09:09:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 1, 2026 - 09:09:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 1, 2026 - 09:09:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:09:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 09:12:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666672 41.822796 41.535473 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.065430 136.068359 136.056641 +util 62.000000 66.000000 55.000000 +[May 1, 2026 - 09:12:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 550 : avg iteration time 160.48 seconds +[May 1, 2026 - 09:12:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 550 : data loading time 0.00 +[May 1, 2026 - 09:12:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 550 : avg forward pass time 71.62 +[May 1, 2026 - 09:12:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 550 : backward pass time 8.42 +[May 1, 2026 - 09:12:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 550 : optimizer step time 0.40 +[May 1, 2026 - 09:12:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 550-------------------- +[May 1, 2026 - 09:12:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 16001.8347 iter count: 1.0 +[May 1, 2026 - 09:12:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.4796 iter count: 1.0 +[May 1, 2026 - 09:12:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 16001.8347iter count: 1.0 +[May 1, 2026 - 09:12:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:12:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 3.9427iter count: 1.0 +[May 1, 2026 - 09:12:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.7948 iter count: 1.0 +[May 1, 2026 - 09:12:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 6.4385 iter count: 1.0 +[May 1, 2026 - 09:12:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 37376.0078iter count: 1.0 +[May 1, 2026 - 09:12:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 4.9015 iter count: 1.0 +[May 1, 2026 - 09:12:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 391201.0625iter count: 1.0 +[May 1, 2026 - 09:12:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.4796iter count: 1.0 +[May 1, 2026 - 09:12:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 6.2672 iter count: 1.0 +[May 1, 2026 - 09:12:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 277728.1875iter count: 1.0 +[May 1, 2026 - 09:12:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.4058 iter count: 1.0 +[May 1, 2026 - 09:12:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 49591.3672iter count: 1.0 +[May 1, 2026 - 09:12:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 2.8614 iter count: 1.0 +[May 1, 2026 - 09:12:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 305.7346iter count: 1.0 +[May 1, 2026 - 09:12:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.8270 iter count: 1.0 +[May 1, 2026 - 09:12:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 15584.9707iter count: 1.0 +[May 1, 2026 - 09:12:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[May 1, 2026 - 09:12:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:12:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 09:13:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666671 41.822796 41.535461 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.432129 67.484375 67.275391 +util 97.750000 100.000000 93.000000 +[May 1, 2026 - 09:13:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 551 : avg iteration time 45.07 seconds +[May 1, 2026 - 09:13:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 551 : data loading time 0.00 +[May 1, 2026 - 09:13:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 551 : avg forward pass time 11.61 +[May 1, 2026 - 09:13:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 551 : backward pass time 10.13 +[May 1, 2026 - 09:13:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 551 : optimizer step time 1.58 +[May 1, 2026 - 09:13:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 551-------------------- +[May 1, 2026 - 09:13:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0117 iter count: 1.0 +[May 1, 2026 - 09:13:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[May 1, 2026 - 09:13:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:13:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 09:13:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666933 41.823818 41.535473 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.534668 68.037109 67.367188 +util 95.000000 100.000000 86.000000 +[May 1, 2026 - 09:13:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 552 : avg iteration time 46.21 seconds +[May 1, 2026 - 09:13:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 552 : data loading time 0.00 +[May 1, 2026 - 09:13:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 552 : avg forward pass time 12.18 +[May 1, 2026 - 09:13:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 552 : backward pass time 10.13 +[May 1, 2026 - 09:13:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 552 : optimizer step time 1.58 +[May 1, 2026 - 09:13:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 552-------------------- +[May 1, 2026 - 09:13:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0114 iter count: 1.0 +[May 1, 2026 - 09:13:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 1, 2026 - 09:13:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:13:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 09:14:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666991 41.824070 41.535477 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.441895 67.724609 67.347656 +util 92.750000 99.000000 88.000000 +[May 1, 2026 - 09:14:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 553 : avg iteration time 46.22 seconds +[May 1, 2026 - 09:14:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 553 : data loading time 0.00 +[May 1, 2026 - 09:14:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 553 : avg forward pass time 12.19 +[May 1, 2026 - 09:14:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 553 : backward pass time 10.14 +[May 1, 2026 - 09:14:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 553 : optimizer step time 1.55 +[May 1, 2026 - 09:14:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 553-------------------- +[May 1, 2026 - 09:14:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0113 iter count: 1.0 +[May 1, 2026 - 09:14:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 1, 2026 - 09:14:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:14:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 09:15:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.667129 41.824543 41.535484 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.549316 68.037109 67.386719 +util 93.500000 99.000000 89.000000 +[May 1, 2026 - 09:15:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 554 : avg iteration time 46.21 seconds +[May 1, 2026 - 09:15:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 554 : data loading time 0.00 +[May 1, 2026 - 09:15:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 554 : avg forward pass time 12.17 +[May 1, 2026 - 09:15:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 554 : backward pass time 10.14 +[May 1, 2026 - 09:15:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 554 : optimizer step time 1.58 +[May 1, 2026 - 09:15:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 554-------------------- +[May 1, 2026 - 09:15:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0104 iter count: 1.0 +[May 1, 2026 - 09:15:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 1, 2026 - 09:15:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:15:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 09:18:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666633 41.822556 41.535500 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.032715 136.095703 136.011719 +util 65.000000 73.000000 59.000000 +[May 1, 2026 - 09:18:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 555 : avg iteration time 165.82 seconds +[May 1, 2026 - 09:18:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 555 : data loading time 0.00 +[May 1, 2026 - 09:18:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 555 : avg forward pass time 74.31 +[May 1, 2026 - 09:18:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 555 : backward pass time 8.42 +[May 1, 2026 - 09:18:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 555 : optimizer step time 0.39 +[May 1, 2026 - 09:18:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 555-------------------- +[May 1, 2026 - 09:18:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 236568.4648 iter count: 1.0 +[May 1, 2026 - 09:18:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5139 iter count: 1.0 +[May 1, 2026 - 09:18:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 236568.4648iter count: 1.0 +[May 1, 2026 - 09:18:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:18:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.0970iter count: 1.0 +[May 1, 2026 - 09:18:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.4961 iter count: 1.0 +[May 1, 2026 - 09:18:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.0371 iter count: 1.0 +[May 1, 2026 - 09:18:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 393788.8125iter count: 1.0 +[May 1, 2026 - 09:18:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 2.6974 iter count: 1.0 +[May 1, 2026 - 09:18:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 9570572.0000iter count: 1.0 +[May 1, 2026 - 09:18:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5139iter count: 1.0 +[May 1, 2026 - 09:18:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 2.7246 iter count: 1.0 +[May 1, 2026 - 09:18:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 232.5879iter count: 1.0 +[May 1, 2026 - 09:18:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.1228 iter count: 1.0 +[May 1, 2026 - 09:18:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 3810.7695iter count: 1.0 +[May 1, 2026 - 09:18:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 6.0335 iter count: 1.0 +[May 1, 2026 - 09:18:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 174032.0312iter count: 1.0 +[May 1, 2026 - 09:18:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.7138 iter count: 1.0 +[May 1, 2026 - 09:18:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 12426.8193iter count: 1.0 +[May 1, 2026 - 09:18:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 1, 2026 - 09:18:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:18:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 09:19:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666612 41.822533 41.535484 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.852051 67.972656 67.490234 +util 92.750000 99.000000 89.000000 +[May 1, 2026 - 09:19:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 556 : avg iteration time 47.82 seconds +[May 1, 2026 - 09:19:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 556 : data loading time 0.00 +[May 1, 2026 - 09:19:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 556 : avg forward pass time 12.97 +[May 1, 2026 - 09:19:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 556 : backward pass time 10.14 +[May 1, 2026 - 09:19:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 556 : optimizer step time 1.59 +[May 1, 2026 - 09:19:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 556-------------------- +[May 1, 2026 - 09:19:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0119 iter count: 1.0 +[May 1, 2026 - 09:19:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[May 1, 2026 - 09:19:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:19:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 09:19:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666811 41.823330 41.535484 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.852051 67.855469 67.841797 +util 94.250000 99.000000 90.000000 +[May 1, 2026 - 09:19:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 557 : avg iteration time 49.03 seconds +[May 1, 2026 - 09:19:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 557 : data loading time 0.00 +[May 1, 2026 - 09:19:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 557 : avg forward pass time 13.59 +[May 1, 2026 - 09:19:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 557 : backward pass time 10.14 +[May 1, 2026 - 09:19:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 557 : optimizer step time 1.57 +[May 1, 2026 - 09:19:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 557-------------------- +[May 1, 2026 - 09:19:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0121 iter count: 1.0 +[May 1, 2026 - 09:19:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[May 1, 2026 - 09:19:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:19:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 09:20:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.667194 41.824810 41.535507 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.705566 67.816406 67.373047 +util 94.000000 99.000000 90.000000 +[May 1, 2026 - 09:20:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 558 : avg iteration time 43.58 seconds +[May 1, 2026 - 09:20:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 558 : data loading time 0.00 +[May 1, 2026 - 09:20:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 558 : avg forward pass time 10.87 +[May 1, 2026 - 09:20:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 558 : backward pass time 10.13 +[May 1, 2026 - 09:20:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 558 : optimizer step time 1.55 +[May 1, 2026 - 09:20:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 558-------------------- +[May 1, 2026 - 09:20:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0124 iter count: 1.0 +[May 1, 2026 - 09:20:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0124 iter count: 1.0 +[May 1, 2026 - 09:20:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:20:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 09:21:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.667252 41.825039 41.535507 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.764160 67.894531 67.373047 +util 96.250000 100.000000 90.000000 +[May 1, 2026 - 09:21:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 559 : avg iteration time 46.28 seconds +[May 1, 2026 - 09:21:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 559 : data loading time 0.00 +[May 1, 2026 - 09:21:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 559 : avg forward pass time 12.20 +[May 1, 2026 - 09:21:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 559 : backward pass time 10.15 +[May 1, 2026 - 09:21:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 559 : optimizer step time 1.58 +[May 1, 2026 - 09:21:21 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 559-------------------- +[May 1, 2026 - 09:21:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0112 iter count: 1.0 +[May 1, 2026 - 09:21:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[May 1, 2026 - 09:21:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:21:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 09:24:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666578 41.822376 41.535496 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.071777 136.076172 136.070312 +util 62.500000 68.000000 58.000000 +[May 1, 2026 - 09:24:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 560 : avg iteration time 163.32 seconds +[May 1, 2026 - 09:24:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 560 : data loading time 0.00 +[May 1, 2026 - 09:24:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 560 : avg forward pass time 73.04 +[May 1, 2026 - 09:24:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 560 : backward pass time 8.44 +[May 1, 2026 - 09:24:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 560 : optimizer step time 0.39 +[May 1, 2026 - 09:24:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 560-------------------- +[May 1, 2026 - 09:24:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 392835.0697 iter count: 1.0 +[May 1, 2026 - 09:24:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5074 iter count: 1.0 +[May 1, 2026 - 09:24:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 392835.0697iter count: 1.0 +[May 1, 2026 - 09:24:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:24:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.6252iter count: 1.0 +[May 1, 2026 - 09:24:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.4872 iter count: 1.0 +[May 1, 2026 - 09:24:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.4998 iter count: 1.0 +[May 1, 2026 - 09:24:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 835416.1875iter count: 1.0 +[May 1, 2026 - 09:24:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 19.5782iter count: 1.0 +[May 1, 2026 - 09:24:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 24146200.0000iter count: 1.0 +[May 1, 2026 - 09:24:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5074iter count: 1.0 +[May 1, 2026 - 09:24:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.5115 iter count: 1.0 +[May 1, 2026 - 09:24:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 1122.0450iter count: 1.0 +[May 1, 2026 - 09:24:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 6.5635 iter count: 1.0 +[May 1, 2026 - 09:24:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 502305.4062iter count: 1.0 +[May 1, 2026 - 09:24:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.7275 iter count: 1.0 +[May 1, 2026 - 09:24:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 1728.5399iter count: 1.0 +[May 1, 2026 - 09:24:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.5869 iter count: 1.0 +[May 1, 2026 - 09:24:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 71244.7734iter count: 1.0 +[May 1, 2026 - 09:24:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 1, 2026 - 09:24:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:24:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 09:24:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666588 41.822399 41.535500 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.568848 67.601562 67.470703 +util 97.750000 100.000000 92.000000 +[May 1, 2026 - 09:24:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 561 : avg iteration time 48.12 seconds +[May 1, 2026 - 09:24:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 561 : data loading time 0.00 +[May 1, 2026 - 09:24:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 561 : avg forward pass time 13.13 +[May 1, 2026 - 09:24:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 561 : backward pass time 10.15 +[May 1, 2026 - 09:24:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 561 : optimizer step time 1.57 +[May 1, 2026 - 09:24:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 561-------------------- +[May 1, 2026 - 09:24:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0115 iter count: 1.0 +[May 1, 2026 - 09:24:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[May 1, 2026 - 09:24:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:24:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 09:25:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666835 41.823383 41.535500 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.549316 67.562500 67.509766 +util 96.750000 100.000000 93.000000 +[May 1, 2026 - 09:25:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 562 : avg iteration time 48.93 seconds +[May 1, 2026 - 09:25:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 562 : data loading time 0.00 +[May 1, 2026 - 09:25:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 562 : avg forward pass time 13.53 +[May 1, 2026 - 09:25:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 562 : backward pass time 10.16 +[May 1, 2026 - 09:25:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 562 : optimizer step time 1.57 +[May 1, 2026 - 09:25:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 562-------------------- +[May 1, 2026 - 09:25:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0113 iter count: 1.0 +[May 1, 2026 - 09:25:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 1, 2026 - 09:25:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:25:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 09:26:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666842 41.823391 41.535507 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.524902 67.562500 67.412109 +util 95.000000 99.000000 91.000000 +[May 1, 2026 - 09:26:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 563 : avg iteration time 43.57 seconds +[May 1, 2026 - 09:26:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 563 : data loading time 0.00 +[May 1, 2026 - 09:26:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 563 : avg forward pass time 10.86 +[May 1, 2026 - 09:26:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 563 : backward pass time 10.14 +[May 1, 2026 - 09:26:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 563 : optimizer step time 1.58 +[May 1, 2026 - 09:26:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 563-------------------- +[May 1, 2026 - 09:26:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0124 iter count: 1.0 +[May 1, 2026 - 09:26:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0124 iter count: 1.0 +[May 1, 2026 - 09:26:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:26:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 09:27:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666984 41.823898 41.535530 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.573730 67.607422 67.562500 +util 95.750000 100.000000 92.000000 +[May 1, 2026 - 09:27:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 564 : avg iteration time 46.21 seconds +[May 1, 2026 - 09:27:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 564 : data loading time 0.00 +[May 1, 2026 - 09:27:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 564 : avg forward pass time 12.18 +[May 1, 2026 - 09:27:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 564 : backward pass time 10.14 +[May 1, 2026 - 09:27:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 564 : optimizer step time 1.57 +[May 1, 2026 - 09:27:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 564-------------------- +[May 1, 2026 - 09:27:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 09:27:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[May 1, 2026 - 09:27:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:27:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 09:29:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666549 41.822197 41.535530 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.086426 136.089844 136.076172 +util 64.750000 73.000000 60.000000 +[May 1, 2026 - 09:29:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 565 : avg iteration time 163.16 seconds +[May 1, 2026 - 09:29:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 565 : data loading time 0.00 +[May 1, 2026 - 09:29:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 565 : avg forward pass time 73.00 +[May 1, 2026 - 09:29:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 565 : backward pass time 8.39 +[May 1, 2026 - 09:29:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 565 : optimizer step time 0.40 +[May 1, 2026 - 09:29:54 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 565-------------------- +[May 1, 2026 - 09:29:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 273233.8125 iter count: 1.0 +[May 1, 2026 - 09:29:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5009 iter count: 1.0 +[May 1, 2026 - 09:29:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 273233.8125iter count: 1.0 +[May 1, 2026 - 09:29:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:29:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 5.0334iter count: 1.0 +[May 1, 2026 - 09:29:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 2.6398 iter count: 1.0 +[May 1, 2026 - 09:29:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.9441 iter count: 1.0 +[May 1, 2026 - 09:29:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 526883.0000iter count: 1.0 +[May 1, 2026 - 09:29:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 196.3004iter count: 1.0 +[May 1, 2026 - 09:29:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 7946936.5000iter count: 1.0 +[May 1, 2026 - 09:29:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5009iter count: 1.0 +[May 1, 2026 - 09:29:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.7901 iter count: 1.0 +[May 1, 2026 - 09:29:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 14475.4541iter count: 1.0 +[May 1, 2026 - 09:29:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 6.9726 iter count: 1.0 +[May 1, 2026 - 09:29:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 1138498.7500iter count: 1.0 +[May 1, 2026 - 09:29:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 7.3596 iter count: 1.0 +[May 1, 2026 - 09:29:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 2468621.7500iter count: 1.0 +[May 1, 2026 - 09:29:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 7.1154 iter count: 1.0 +[May 1, 2026 - 09:29:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 1514928.6250iter count: 1.0 +[May 1, 2026 - 09:29:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[May 1, 2026 - 09:29:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:29:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 09:30:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666609 41.822437 41.535507 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.642090 67.998047 67.523438 +util 93.750000 99.000000 89.000000 +[May 1, 2026 - 09:30:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 566 : avg iteration time 47.81 seconds +[May 1, 2026 - 09:30:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 566 : data loading time 0.00 +[May 1, 2026 - 09:30:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 566 : avg forward pass time 12.98 +[May 1, 2026 - 09:30:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 566 : backward pass time 10.14 +[May 1, 2026 - 09:30:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 566 : optimizer step time 1.57 +[May 1, 2026 - 09:30:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 566-------------------- +[May 1, 2026 - 09:30:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 09:30:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[May 1, 2026 - 09:30:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:30:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 09:31:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666862 41.823441 41.535507 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.539551 67.998047 67.386719 +util 93.750000 99.000000 89.000000 +[May 1, 2026 - 09:31:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 567 : avg iteration time 43.60 seconds +[May 1, 2026 - 09:31:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 567 : data loading time 0.00 +[May 1, 2026 - 09:31:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 567 : avg forward pass time 10.87 +[May 1, 2026 - 09:31:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 567 : backward pass time 10.14 +[May 1, 2026 - 09:31:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 567 : optimizer step time 1.57 +[May 1, 2026 - 09:31:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 567-------------------- +[May 1, 2026 - 09:31:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0116 iter count: 1.0 +[May 1, 2026 - 09:31:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[May 1, 2026 - 09:31:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:31:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 09:32:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666864 41.823452 41.535507 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.559082 68.017578 67.406250 +util 94.750000 99.000000 88.000000 +[May 1, 2026 - 09:32:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 568 : avg iteration time 46.25 seconds +[May 1, 2026 - 09:32:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 568 : data loading time 0.00 +[May 1, 2026 - 09:32:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 568 : avg forward pass time 12.18 +[May 1, 2026 - 09:32:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 568 : backward pass time 10.15 +[May 1, 2026 - 09:32:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 568 : optimizer step time 1.57 +[May 1, 2026 - 09:32:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 568-------------------- +[May 1, 2026 - 09:32:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 1, 2026 - 09:32:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 1, 2026 - 09:32:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:32:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 09:32:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666854 41.823395 41.535530 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.622559 67.978516 67.503906 +util 92.500000 99.000000 88.000000 +[May 1, 2026 - 09:32:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 569 : avg iteration time 46.28 seconds +[May 1, 2026 - 09:32:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 569 : data loading time 0.00 +[May 1, 2026 - 09:32:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 569 : avg forward pass time 12.21 +[May 1, 2026 - 09:32:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 569 : backward pass time 10.15 +[May 1, 2026 - 09:32:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 569 : optimizer step time 1.57 +[May 1, 2026 - 09:32:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 569-------------------- +[May 1, 2026 - 09:32:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[May 1, 2026 - 09:32:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[May 1, 2026 - 09:32:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:32:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 09:35:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666711 41.822823 41.535511 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.027832 136.031250 136.017578 +util 68.500000 73.000000 62.000000 +[May 1, 2026 - 09:35:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 570 : avg iteration time 165.99 seconds +[May 1, 2026 - 09:35:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 570 : data loading time 0.00 +[May 1, 2026 - 09:35:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 570 : avg forward pass time 74.38 +[May 1, 2026 - 09:35:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 570 : backward pass time 8.45 +[May 1, 2026 - 09:35:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 570 : optimizer step time 0.39 +[May 1, 2026 - 09:35:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 570-------------------- +[May 1, 2026 - 09:35:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 542848.5088 iter count: 1.0 +[May 1, 2026 - 09:35:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5769 iter count: 1.0 +[May 1, 2026 - 09:35:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 542848.5088iter count: 1.0 +[May 1, 2026 - 09:35:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:35:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.7107iter count: 1.0 +[May 1, 2026 - 09:35:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.6331 iter count: 1.0 +[May 1, 2026 - 09:35:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.2286 iter count: 1.0 +[May 1, 2026 - 09:35:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 901161.7500iter count: 1.0 +[May 1, 2026 - 09:35:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 3.5471 iter count: 1.0 +[May 1, 2026 - 09:35:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 14037689.0000iter count: 1.0 +[May 1, 2026 - 09:35:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5769iter count: 1.0 +[May 1, 2026 - 09:35:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.8078 iter count: 1.0 +[May 1, 2026 - 09:35:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 2029.7643iter count: 1.0 +[May 1, 2026 - 09:35:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 6.1324 iter count: 1.0 +[May 1, 2026 - 09:35:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 212106.5312iter count: 1.0 +[May 1, 2026 - 09:35:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 2.4712 iter count: 1.0 +[May 1, 2026 - 09:35:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 140.1187iter count: 1.0 +[May 1, 2026 - 09:35:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 2.8826 iter count: 1.0 +[May 1, 2026 - 09:35:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 318.9772iter count: 1.0 +[May 1, 2026 - 09:35:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 1, 2026 - 09:35:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:35:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 09:36:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666718 41.822823 41.535511 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.583984 68.058594 67.425781 +util 97.000000 100.000000 93.000000 +[May 1, 2026 - 09:36:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 571 : avg iteration time 50.94 seconds +[May 1, 2026 - 09:36:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 571 : data loading time 0.00 +[May 1, 2026 - 09:36:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 571 : avg forward pass time 14.52 +[May 1, 2026 - 09:36:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 571 : backward pass time 10.15 +[May 1, 2026 - 09:36:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 571 : optimizer step time 1.60 +[May 1, 2026 - 09:36:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 571-------------------- +[May 1, 2026 - 09:36:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0126 iter count: 1.0 +[May 1, 2026 - 09:36:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0126 iter count: 1.0 +[May 1, 2026 - 09:36:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:36:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 09:37:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666980 41.823803 41.535568 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.657227 68.058594 67.523438 +util 94.750000 100.000000 87.000000 +[May 1, 2026 - 09:37:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 572 : avg iteration time 43.72 seconds +[May 1, 2026 - 09:37:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 572 : data loading time 0.00 +[May 1, 2026 - 09:37:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 572 : avg forward pass time 10.89 +[May 1, 2026 - 09:37:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 572 : backward pass time 10.15 +[May 1, 2026 - 09:37:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 572 : optimizer step time 1.64 +[May 1, 2026 - 09:37:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 572-------------------- +[May 1, 2026 - 09:37:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0119 iter count: 1.0 +[May 1, 2026 - 09:37:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[May 1, 2026 - 09:37:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:37:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 09:38:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.667043 41.824062 41.535545 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.530273 68.019531 67.367188 +util 97.000000 100.000000 91.000000 +[May 1, 2026 - 09:38:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 573 : avg iteration time 46.52 seconds +[May 1, 2026 - 09:38:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 573 : data loading time 0.00 +[May 1, 2026 - 09:38:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 573 : avg forward pass time 12.33 +[May 1, 2026 - 09:38:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 573 : backward pass time 10.15 +[May 1, 2026 - 09:38:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 573 : optimizer step time 1.57 +[May 1, 2026 - 09:38:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 573-------------------- +[May 1, 2026 - 09:38:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 09:38:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 09:38:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:38:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 09:38:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.667035 41.824062 41.535553 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.544922 68.019531 67.386719 +util 95.750000 99.000000 89.000000 +[May 1, 2026 - 09:38:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 574 : avg iteration time 43.58 seconds +[May 1, 2026 - 09:38:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 574 : data loading time 0.00 +[May 1, 2026 - 09:38:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 574 : avg forward pass time 10.85 +[May 1, 2026 - 09:38:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 574 : backward pass time 10.16 +[May 1, 2026 - 09:38:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 574 : optimizer step time 1.57 +[May 1, 2026 - 09:38:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 574-------------------- +[May 1, 2026 - 09:38:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0122 iter count: 1.0 +[May 1, 2026 - 09:38:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0122 iter count: 1.0 +[May 1, 2026 - 09:38:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:38:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 09:41:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666963 41.823624 41.535583 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.037598 136.056641 136.031250 +util 65.250000 70.000000 58.000000 +[May 1, 2026 - 09:41:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 575 : avg iteration time 168.42 seconds +[May 1, 2026 - 09:41:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 575 : data loading time 0.00 +[May 1, 2026 - 09:41:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 575 : avg forward pass time 75.62 +[May 1, 2026 - 09:41:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 575 : backward pass time 8.38 +[May 1, 2026 - 09:41:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 575 : optimizer step time 0.40 +[May 1, 2026 - 09:41:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 575-------------------- +[May 1, 2026 - 09:41:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 108891.8726 iter count: 1.0 +[May 1, 2026 - 09:41:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5248 iter count: 1.0 +[May 1, 2026 - 09:41:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 108891.8726iter count: 1.0 +[May 1, 2026 - 09:41:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:41:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.8672iter count: 1.0 +[May 1, 2026 - 09:41:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.2284 iter count: 1.0 +[May 1, 2026 - 09:41:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.1775 iter count: 1.0 +[May 1, 2026 - 09:41:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 217421.5938iter count: 1.0 +[May 1, 2026 - 09:41:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 11.6677iter count: 1.0 +[May 1, 2026 - 09:41:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 1715179.2500iter count: 1.0 +[May 1, 2026 - 09:41:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5248iter count: 1.0 +[May 1, 2026 - 09:41:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.6042 iter count: 1.0 +[May 1, 2026 - 09:41:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 1350.7279iter count: 1.0 +[May 1, 2026 - 09:41:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.8714 iter count: 1.0 +[May 1, 2026 - 09:41:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 125843.2266iter count: 1.0 +[May 1, 2026 - 09:41:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.3343 iter count: 1.0 +[May 1, 2026 - 09:41:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 42983.4141iter count: 1.0 +[May 1, 2026 - 09:41:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.0934 iter count: 1.0 +[May 1, 2026 - 09:41:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 3592.9521iter count: 1.0 +[May 1, 2026 - 09:41:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0118 iter count: 1.0 +[May 1, 2026 - 09:41:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:41:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 09:42:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666970 41.823631 41.535561 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.397949 67.431641 67.386719 +util 95.500000 99.000000 89.000000 +[May 1, 2026 - 09:42:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 576 : avg iteration time 50.57 seconds +[May 1, 2026 - 09:42:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 576 : data loading time 0.00 +[May 1, 2026 - 09:42:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 576 : avg forward pass time 14.36 +[May 1, 2026 - 09:42:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 576 : backward pass time 10.15 +[May 1, 2026 - 09:42:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 576 : optimizer step time 1.58 +[May 1, 2026 - 09:42:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 576-------------------- +[May 1, 2026 - 09:42:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[May 1, 2026 - 09:42:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[May 1, 2026 - 09:42:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:42:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 09:43:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.667159 41.824387 41.535606 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.637207 68.095703 67.484375 +util 92.250000 98.000000 88.000000 +[May 1, 2026 - 09:43:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 577 : avg iteration time 43.51 seconds +[May 1, 2026 - 09:43:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 577 : data loading time 0.00 +[May 1, 2026 - 09:43:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 577 : avg forward pass time 10.84 +[May 1, 2026 - 09:43:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 577 : backward pass time 10.13 +[May 1, 2026 - 09:43:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 577 : optimizer step time 1.57 +[May 1, 2026 - 09:43:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 577-------------------- +[May 1, 2026 - 09:43:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0118 iter count: 1.0 +[May 1, 2026 - 09:43:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0118 iter count: 1.0 +[May 1, 2026 - 09:43:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:43:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 09:43:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.667165 41.824432 41.535561 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.520020 67.626953 67.484375 +util 96.500000 99.000000 93.000000 +[May 1, 2026 - 09:43:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 578 : avg iteration time 46.26 seconds +[May 1, 2026 - 09:43:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 578 : data loading time 0.00 +[May 1, 2026 - 09:43:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 578 : avg forward pass time 12.20 +[May 1, 2026 - 09:43:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 578 : backward pass time 10.14 +[May 1, 2026 - 09:43:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 578 : optimizer step time 1.57 +[May 1, 2026 - 09:43:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 578-------------------- +[May 1, 2026 - 09:43:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0121 iter count: 1.0 +[May 1, 2026 - 09:43:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[May 1, 2026 - 09:43:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:43:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 09:44:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.667149 41.824390 41.535561 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.520020 67.626953 67.484375 +util 97.500000 99.000000 93.000000 +[May 1, 2026 - 09:44:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 579 : avg iteration time 46.28 seconds +[May 1, 2026 - 09:44:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 579 : data loading time 0.00 +[May 1, 2026 - 09:44:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 579 : avg forward pass time 12.22 +[May 1, 2026 - 09:44:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 579 : backward pass time 10.12 +[May 1, 2026 - 09:44:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 579 : optimizer step time 1.57 +[May 1, 2026 - 09:44:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 579-------------------- +[May 1, 2026 - 09:44:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 09:44:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[May 1, 2026 - 09:44:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:44:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 09:47:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.667002 41.823242 41.535793 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.042480 136.076172 136.031250 +util 67.250000 72.000000 61.000000 +[May 1, 2026 - 09:47:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 580 : avg iteration time 165.78 seconds +[May 1, 2026 - 09:47:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 580 : data loading time 0.00 +[May 1, 2026 - 09:47:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 580 : avg forward pass time 74.28 +[May 1, 2026 - 09:47:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 580 : backward pass time 8.43 +[May 1, 2026 - 09:47:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 580 : optimizer step time 0.39 +[May 1, 2026 - 09:47:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 580-------------------- +[May 1, 2026 - 09:47:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 525803.3386 iter count: 1.0 +[May 1, 2026 - 09:47:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5350 iter count: 1.0 +[May 1, 2026 - 09:47:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 525803.3386iter count: 1.0 +[May 1, 2026 - 09:47:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:47:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.3165iter count: 1.0 +[May 1, 2026 - 09:47:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.1649 iter count: 1.0 +[May 1, 2026 - 09:47:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.6519 iter count: 1.0 +[May 1, 2026 - 09:47:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 1093648.5000iter count: 1.0 +[May 1, 2026 - 09:47:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 10.2758iter count: 1.0 +[May 1, 2026 - 09:47:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 32732492.0000iter count: 1.0 +[May 1, 2026 - 09:47:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5350iter count: 1.0 +[May 1, 2026 - 09:47:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.5884 iter count: 1.0 +[May 1, 2026 - 09:47:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 1308.7609iter count: 1.0 +[May 1, 2026 - 09:47:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 6.8668 iter count: 1.0 +[May 1, 2026 - 09:47:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 921400.5625iter count: 1.0 +[May 1, 2026 - 09:47:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.2336 iter count: 1.0 +[May 1, 2026 - 09:47:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 4755.9678iter count: 1.0 +[May 1, 2026 - 09:47:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.4387 iter count: 1.0 +[May 1, 2026 - 09:47:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 7167.5244iter count: 1.0 +[May 1, 2026 - 09:47:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0122 iter count: 1.0 +[May 1, 2026 - 09:47:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:47:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 09:48:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666999 41.823231 41.535793 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.480957 67.646484 67.425781 +util 93.500000 99.000000 89.000000 +[May 1, 2026 - 09:48:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 581 : avg iteration time 47.85 seconds +[May 1, 2026 - 09:48:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 581 : data loading time 0.00 +[May 1, 2026 - 09:48:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 581 : avg forward pass time 13.00 +[May 1, 2026 - 09:48:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 581 : backward pass time 10.13 +[May 1, 2026 - 09:48:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 581 : optimizer step time 1.58 +[May 1, 2026 - 09:48:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 581-------------------- +[May 1, 2026 - 09:48:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0117 iter count: 1.0 +[May 1, 2026 - 09:48:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[May 1, 2026 - 09:48:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:48:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 09:49:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.667122 41.823730 41.535793 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.603027 67.958984 67.484375 +util 97.000000 100.000000 91.000000 +[May 1, 2026 - 09:49:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 582 : avg iteration time 43.55 seconds +[May 1, 2026 - 09:49:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 582 : data loading time 0.00 +[May 1, 2026 - 09:49:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 582 : avg forward pass time 10.85 +[May 1, 2026 - 09:49:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 582 : backward pass time 10.15 +[May 1, 2026 - 09:49:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 582 : optimizer step time 1.57 +[May 1, 2026 - 09:49:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 582-------------------- +[May 1, 2026 - 09:49:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0132 iter count: 1.0 +[May 1, 2026 - 09:49:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0132 iter count: 1.0 +[May 1, 2026 - 09:49:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:49:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 09:49:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.667197 41.823982 41.535816 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.529785 67.958984 67.386719 +util 93.250000 100.000000 89.000000 +[May 1, 2026 - 09:49:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 583 : avg iteration time 46.25 seconds +[May 1, 2026 - 09:49:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 583 : data loading time 0.00 +[May 1, 2026 - 09:49:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 583 : avg forward pass time 12.20 +[May 1, 2026 - 09:49:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 583 : backward pass time 10.15 +[May 1, 2026 - 09:49:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 583 : optimizer step time 1.55 +[May 1, 2026 - 09:49:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 583-------------------- +[May 1, 2026 - 09:49:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0119 iter count: 1.0 +[May 1, 2026 - 09:49:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[May 1, 2026 - 09:49:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:49:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 09:50:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.667194 41.823982 41.535793 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.529785 67.958984 67.386719 +util 94.250000 98.000000 87.000000 +[May 1, 2026 - 09:50:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 584 : avg iteration time 46.31 seconds +[May 1, 2026 - 09:50:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 584 : data loading time 0.00 +[May 1, 2026 - 09:50:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 584 : avg forward pass time 12.21 +[May 1, 2026 - 09:50:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 584 : backward pass time 10.15 +[May 1, 2026 - 09:50:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 584 : optimizer step time 1.57 +[May 1, 2026 - 09:50:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 584-------------------- +[May 1, 2026 - 09:50:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0114 iter count: 1.0 +[May 1, 2026 - 09:50:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 1, 2026 - 09:50:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:50:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 09:53:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666787 41.822388 41.535793 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.013184 136.017578 136.011719 +util 64.750000 73.000000 59.000000 +[May 1, 2026 - 09:53:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 585 : avg iteration time 168.72 seconds +[May 1, 2026 - 09:53:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 585 : data loading time 0.00 +[May 1, 2026 - 09:53:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 585 : avg forward pass time 75.74 +[May 1, 2026 - 09:53:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 585 : backward pass time 8.42 +[May 1, 2026 - 09:53:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 585 : optimizer step time 0.39 +[May 1, 2026 - 09:53:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 585-------------------- +[May 1, 2026 - 09:53:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 33488.1987 iter count: 1.0 +[May 1, 2026 - 09:53:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5892 iter count: 1.0 +[May 1, 2026 - 09:53:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 33488.1987iter count: 1.0 +[May 1, 2026 - 09:53:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:53:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.2439iter count: 1.0 +[May 1, 2026 - 09:53:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.8242 iter count: 1.0 +[May 1, 2026 - 09:53:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 6.7701 iter count: 1.0 +[May 1, 2026 - 09:53:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 65994.5078iter count: 1.0 +[May 1, 2026 - 09:53:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 5.1985 iter count: 1.0 +[May 1, 2026 - 09:53:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 759279.9375iter count: 1.0 +[May 1, 2026 - 09:53:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5892iter count: 1.0 +[May 1, 2026 - 09:53:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.9988 iter count: 1.0 +[May 1, 2026 - 09:53:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 21973.2168iter count: 1.0 +[May 1, 2026 - 09:53:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.2612 iter count: 1.0 +[May 1, 2026 - 09:53:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 5025.9673iter count: 1.0 +[May 1, 2026 - 09:53:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 6.2547 iter count: 1.0 +[May 1, 2026 - 09:53:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 270897.2188iter count: 1.0 +[May 1, 2026 - 09:53:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.1567 iter count: 1.0 +[May 1, 2026 - 09:53:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 222654.4844iter count: 1.0 +[May 1, 2026 - 09:53:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[May 1, 2026 - 09:53:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:53:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 09:54:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666846 41.822617 41.535801 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.383301 67.431641 67.367188 +util 97.000000 100.000000 91.000000 +[May 1, 2026 - 09:54:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 586 : avg iteration time 45.13 seconds +[May 1, 2026 - 09:54:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 586 : data loading time 0.00 +[May 1, 2026 - 09:54:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 586 : avg forward pass time 11.65 +[May 1, 2026 - 09:54:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 586 : backward pass time 10.12 +[May 1, 2026 - 09:54:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 586 : optimizer step time 1.55 +[May 1, 2026 - 09:54:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 586-------------------- +[May 1, 2026 - 09:54:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0121 iter count: 1.0 +[May 1, 2026 - 09:54:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[May 1, 2026 - 09:54:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:54:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 09:54:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.667038 41.823368 41.535809 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.378418 67.470703 67.347656 +util 92.000000 99.000000 87.000000 +[May 1, 2026 - 09:54:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 587 : avg iteration time 43.63 seconds +[May 1, 2026 - 09:54:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 587 : data loading time 0.00 +[May 1, 2026 - 09:54:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 587 : avg forward pass time 10.90 +[May 1, 2026 - 09:54:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 587 : backward pass time 10.14 +[May 1, 2026 - 09:54:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 587 : optimizer step time 1.57 +[May 1, 2026 - 09:54:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 587-------------------- +[May 1, 2026 - 09:54:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0115 iter count: 1.0 +[May 1, 2026 - 09:54:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[May 1, 2026 - 09:54:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:54:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 09:55:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.667041 41.823372 41.535801 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.441895 67.841797 67.308594 +util 96.500000 100.000000 93.000000 +[May 1, 2026 - 09:55:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 588 : avg iteration time 46.31 seconds +[May 1, 2026 - 09:55:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 588 : data loading time 0.00 +[May 1, 2026 - 09:55:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 588 : avg forward pass time 12.22 +[May 1, 2026 - 09:55:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 588 : backward pass time 10.14 +[May 1, 2026 - 09:55:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 588 : optimizer step time 1.57 +[May 1, 2026 - 09:55:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 588-------------------- +[May 1, 2026 - 09:55:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0115 iter count: 1.0 +[May 1, 2026 - 09:55:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[May 1, 2026 - 09:55:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:55:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 09:56:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.667109 41.823647 41.535801 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.432129 67.464844 67.333984 +util 97.250000 99.000000 94.000000 +[May 1, 2026 - 09:56:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 589 : avg iteration time 46.42 seconds +[May 1, 2026 - 09:56:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 589 : data loading time 0.00 +[May 1, 2026 - 09:56:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 589 : avg forward pass time 12.26 +[May 1, 2026 - 09:56:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 589 : backward pass time 10.15 +[May 1, 2026 - 09:56:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 589 : optimizer step time 1.58 +[May 1, 2026 - 09:56:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 589-------------------- +[May 1, 2026 - 09:56:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0104 iter count: 1.0 +[May 1, 2026 - 09:56:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[May 1, 2026 - 09:56:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:56:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 09:59:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666702 41.821991 41.535831 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.041016 136.048828 136.017578 +util 61.750000 71.000000 56.000000 +[May 1, 2026 - 09:59:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 590 : avg iteration time 163.09 seconds +[May 1, 2026 - 09:59:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 590 : data loading time 0.00 +[May 1, 2026 - 09:59:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 590 : avg forward pass time 72.97 +[May 1, 2026 - 09:59:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 590 : backward pass time 8.42 +[May 1, 2026 - 09:59:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 590 : optimizer step time 0.39 +[May 1, 2026 - 09:59:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 590-------------------- +[May 1, 2026 - 09:59:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 18236.0325 iter count: 1.0 +[May 1, 2026 - 09:59:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5566 iter count: 1.0 +[May 1, 2026 - 09:59:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 18236.0325iter count: 1.0 +[May 1, 2026 - 09:59:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:59:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.2162iter count: 1.0 +[May 1, 2026 - 09:59:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.6983 iter count: 1.0 +[May 1, 2026 - 09:59:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 6.1524 iter count: 1.0 +[May 1, 2026 - 09:59:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 31351.4883iter count: 1.0 +[May 1, 2026 - 09:59:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 29.8650iter count: 1.0 +[May 1, 2026 - 09:59:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 220752.2188iter count: 1.0 +[May 1, 2026 - 09:59:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5566iter count: 1.0 +[May 1, 2026 - 09:59:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 2.3181 iter count: 1.0 +[May 1, 2026 - 09:59:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 103.1487iter count: 1.0 +[May 1, 2026 - 09:59:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.2261 iter count: 1.0 +[May 1, 2026 - 09:59:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 4685.8066iter count: 1.0 +[May 1, 2026 - 09:59:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.7734 iter count: 1.0 +[May 1, 2026 - 09:59:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 13998.7900iter count: 1.0 +[May 1, 2026 - 09:59:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.6621 iter count: 1.0 +[May 1, 2026 - 09:59:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 11205.6455iter count: 1.0 +[May 1, 2026 - 09:59:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 1, 2026 - 09:59:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:59:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 09:59:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666759 41.822239 41.535812 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.412598 67.490234 67.386719 +util 94.000000 100.000000 90.000000 +[May 1, 2026 - 09:59:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 591 : avg iteration time 47.82 seconds +[May 1, 2026 - 09:59:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 591 : data loading time 0.00 +[May 1, 2026 - 09:59:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 591 : avg forward pass time 12.98 +[May 1, 2026 - 09:59:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 591 : backward pass time 10.15 +[May 1, 2026 - 09:59:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 591 : optimizer step time 1.56 +[May 1, 2026 - 09:59:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 591-------------------- +[May 1, 2026 - 09:59:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0125 iter count: 1.0 +[May 1, 2026 - 09:59:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0125 iter count: 1.0 +[May 1, 2026 - 09:59:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 09:59:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 10:00:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.667020 41.823235 41.535828 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.500488 67.900391 67.367188 +util 96.250000 99.000000 93.000000 +[May 1, 2026 - 10:00:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 592 : avg iteration time 49.08 seconds +[May 1, 2026 - 10:00:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 592 : data loading time 0.00 +[May 1, 2026 - 10:00:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 592 : avg forward pass time 13.61 +[May 1, 2026 - 10:00:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 592 : backward pass time 10.14 +[May 1, 2026 - 10:00:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 592 : optimizer step time 1.56 +[May 1, 2026 - 10:00:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 592-------------------- +[May 1, 2026 - 10:00:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0108 iter count: 1.0 +[May 1, 2026 - 10:00:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[May 1, 2026 - 10:00:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 10:00:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 10:01:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.667020 41.823235 41.535851 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.632324 67.958984 67.523438 +util 95.000000 99.000000 91.000000 +[May 1, 2026 - 10:01:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 593 : avg iteration time 46.18 seconds +[May 1, 2026 - 10:01:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 593 : data loading time 0.00 +[May 1, 2026 - 10:01:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 593 : avg forward pass time 12.19 +[May 1, 2026 - 10:01:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 593 : backward pass time 10.13 +[May 1, 2026 - 10:01:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 593 : optimizer step time 1.55 +[May 1, 2026 - 10:01:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 593-------------------- +[May 1, 2026 - 10:01:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0110 iter count: 1.0 +[May 1, 2026 - 10:01:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[May 1, 2026 - 10:01:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 10:01:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 10:02:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.667020 41.823235 41.535828 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.432129 67.568359 67.386719 +util 93.250000 100.000000 89.000000 +[May 1, 2026 - 10:02:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 594 : avg iteration time 43.54 seconds +[May 1, 2026 - 10:02:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 594 : data loading time 0.00 +[May 1, 2026 - 10:02:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 594 : avg forward pass time 10.84 +[May 1, 2026 - 10:02:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 594 : backward pass time 10.14 +[May 1, 2026 - 10:02:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 594 : optimizer step time 1.57 +[May 1, 2026 - 10:02:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 594-------------------- +[May 1, 2026 - 10:02:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0122 iter count: 1.0 +[May 1, 2026 - 10:02:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0122 iter count: 1.0 +[May 1, 2026 - 10:02:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 10:02:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 10:04:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666706 41.821972 41.535835 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.018066 136.037109 136.011719 +util 67.000000 72.000000 63.000000 +[May 1, 2026 - 10:04:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 595 : avg iteration time 166.10 seconds +[May 1, 2026 - 10:04:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 595 : data loading time 0.00 +[May 1, 2026 - 10:04:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 595 : avg forward pass time 74.42 +[May 1, 2026 - 10:04:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 595 : backward pass time 8.42 +[May 1, 2026 - 10:04:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 595 : optimizer step time 0.39 +[May 1, 2026 - 10:04:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 595-------------------- +[May 1, 2026 - 10:04:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 401283.8950 iter count: 1.0 +[May 1, 2026 - 10:04:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5529 iter count: 1.0 +[May 1, 2026 - 10:04:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 401283.8950iter count: 1.0 +[May 1, 2026 - 10:04:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 10:04:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 5.0313iter count: 1.0 +[May 1, 2026 - 10:04:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 2.0476 iter count: 1.0 +[May 1, 2026 - 10:04:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.2731 iter count: 1.0 +[May 1, 2026 - 10:04:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 886432.7500iter count: 1.0 +[May 1, 2026 - 10:04:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 60.0479iter count: 1.0 +[May 1, 2026 - 10:04:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 15342230.0000iter count: 1.0 +[May 1, 2026 - 10:04:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5529iter count: 1.0 +[May 1, 2026 - 10:04:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 2.6105 iter count: 1.0 +[May 1, 2026 - 10:04:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 185.1148iter count: 1.0 +[May 1, 2026 - 10:04:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 7.5568 iter count: 1.0 +[May 1, 2026 - 10:04:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 3662592.5000iter count: 1.0 +[May 1, 2026 - 10:04:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.0180 iter count: 1.0 +[May 1, 2026 - 10:04:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 22833.8242iter count: 1.0 +[May 1, 2026 - 10:04:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.0508 iter count: 1.0 +[May 1, 2026 - 10:04:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 24383.1211iter count: 1.0 +[May 1, 2026 - 10:04:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[May 1, 2026 - 10:04:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 10:04:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[May 1, 2026 - 10:05:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666698 41.821949 41.535828 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.383301 67.425781 67.255859 +util 94.750000 99.000000 91.000000 +[May 1, 2026 - 10:05:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 596 : avg iteration time 45.16 seconds +[May 1, 2026 - 10:05:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 596 : data loading time 0.00 +[May 1, 2026 - 10:05:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 596 : avg forward pass time 11.65 +[May 1, 2026 - 10:05:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 596 : backward pass time 10.16 +[May 1, 2026 - 10:05:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 596 : optimizer step time 1.57 +[May 1, 2026 - 10:05:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 596-------------------- +[May 1, 2026 - 10:05:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0119 iter count: 1.0 +[May 1, 2026 - 10:05:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[May 1, 2026 - 10:05:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 10:05:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 10:06:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666901 41.822750 41.535831 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.378418 67.445312 67.177734 +util 97.500000 100.000000 91.000000 +[May 1, 2026 - 10:06:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 597 : avg iteration time 43.69 seconds +[May 1, 2026 - 10:06:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 597 : data loading time 0.00 +[May 1, 2026 - 10:06:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 597 : avg forward pass time 10.92 +[May 1, 2026 - 10:06:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 597 : backward pass time 10.14 +[May 1, 2026 - 10:06:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 597 : optimizer step time 1.58 +[May 1, 2026 - 10:06:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 597-------------------- +[May 1, 2026 - 10:06:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0113 iter count: 1.0 +[May 1, 2026 - 10:06:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[May 1, 2026 - 10:06:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 10:06:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 10:07:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.667209 41.823997 41.535839 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.393066 67.412109 67.386719 +util 92.000000 97.000000 88.000000 +[May 1, 2026 - 10:07:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 598 : avg iteration time 48.87 seconds +[May 1, 2026 - 10:07:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 598 : data loading time 0.00 +[May 1, 2026 - 10:07:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 598 : avg forward pass time 13.52 +[May 1, 2026 - 10:07:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 598 : backward pass time 10.13 +[May 1, 2026 - 10:07:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 598 : optimizer step time 1.56 +[May 1, 2026 - 10:07:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 598-------------------- +[May 1, 2026 - 10:07:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0100 iter count: 1.0 +[May 1, 2026 - 10:07:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[May 1, 2026 - 10:07:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 10:07:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 10:08:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.667214 41.823997 41.535839 +peak_gpu_mem_gb 65.143914 65.185156 65.130167 +peak_gpu_mem_reserved_gb 67.852051 67.992188 67.431641 +util 93.500000 99.000000 89.000000 +[May 1, 2026 - 10:08:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 599 : avg iteration time 49.02 seconds +[May 1, 2026 - 10:08:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 599 : data loading time 0.00 +[May 1, 2026 - 10:08:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 599 : avg forward pass time 13.58 +[May 1, 2026 - 10:08:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 599 : backward pass time 10.13 +[May 1, 2026 - 10:08:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 599 : optimizer step time 1.56 +[May 1, 2026 - 10:08:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 599-------------------- +[May 1, 2026 - 10:08:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 10:08:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[May 1, 2026 - 10:08:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[May 1, 2026 - 10:08:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[May 1, 2026 - 10:10:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 41.666533 41.821270 41.535839 +peak_gpu_mem_gb 132.063042 132.104284 132.049294 +peak_gpu_mem_reserved_gb 136.052246 136.056641 136.050781 +util 63.250000 69.000000 59.000000 +[May 1, 2026 - 10:10:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 600 : avg iteration time 165.81 seconds +[May 1, 2026 - 10:10:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 600 : data loading time 0.00 +[May 1, 2026 - 10:10:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 600 : avg forward pass time 74.30 +[May 1, 2026 - 10:10:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 600 : backward pass time 8.42 +[May 1, 2026 - 10:10:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 600 : optimizer step time 0.39 +[May 1, 2026 - 10:10:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 600-------------------- +[May 1, 2026 - 10:10:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 33094.3105 iter count: 1.0 +[May 1, 2026 - 10:10:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5255 iter count: 1.0 +[May 1, 2026 - 10:10:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 33094.3105iter count: 1.0 +[May 1, 2026 - 10:10:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[May 1, 2026 - 10:10:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.1607iter count: 1.0 +[May 1, 2026 - 10:10:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.4610 iter count: 1.0 +[May 1, 2026 - 10:10:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 6.6744 iter count: 1.0 +[May 1, 2026 - 10:10:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 60323.4883iter count: 1.0 +[May 1, 2026 - 10:10:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 2.5142 iter count: 1.0 +[May 1, 2026 - 10:10:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 627062.5625iter count: 1.0 +[May 1, 2026 - 10:10:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5255iter count: 1.0 +[May 1, 2026 - 10:10:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.3941 iter count: 1.0 +[May 1, 2026 - 10:10:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 6556.8604iter count: 1.0 +[May 1, 2026 - 10:10:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.5555 iter count: 1.0 +[May 1, 2026 - 10:10:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 1225.4612iter count: 1.0 +[May 1, 2026 - 10:10:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.8698 iter count: 1.0 +[May 1, 2026 - 10:10:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 2297.6609iter count: 1.0 +[May 1, 2026 - 10:10:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 2.8982 iter count: 1.0 +[May 1, 2026 - 10:10:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 329.1447iter count: 1.0 +[May 1, 2026 - 10:10:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[May 1, 2026 - 10:10:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[May 1, 2026 - 10:11:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 14.93 seconds +[May 1, 2026 - 10:11:07 | INFO | fastgen.trainer:save_checkpoint:266 ] Saving checkpoint iteration 600 +[May 1, 2026 - 10:11:07 | INFO | fastgen.utils.checkpointer:save:311 ] Saving FSDP model to prefix /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints/0000600 +Traceback (most recent call last): diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/requirements.txt b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..a7117b8203d3b4df2b52c765d49a822a97b57cd9 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/requirements.txt @@ -0,0 +1,247 @@ +nvitop==1.6.1 +ftfy==6.3.1 +braceexpand==0.1.7 +antlr4-python3-runtime==4.9.3 +webdataset==1.0.2 +sentry-sdk==2.53.0 +rdkit==2025.9.5 +python-dotenv==1.2.1 +proglog==0.1.12 +omegaconf==2.3.0 +narwhals==2.17.0 +loguru==0.7.3 +imageio-ffmpeg==0.6.0 +plotly==6.5.2 +moviepy==2.2.1 +hydra-core==1.3.2 +toml==0.10.2 +wandb==0.25.0 +types-certifi==2021.10.8.3 +types-toml==0.10.8.20260408 +synchronicity==0.12.2 +propcache==0.4.1 +multidict==6.7.1 +hyperframe==6.1.0 +hpack==4.1.0 +frozenlist==1.8.0 +cbor2==5.9.0 +aiohappyeyeballs==2.6.1 +yarl==1.23.0 +watchfiles==1.1.1 +h2==4.3.0 +aiosignal==1.4.0 +grpclib==0.4.9 +aiohttp==3.13.5 +modal==1.4.1 +packaging==25.0 +setuptools==80.10.2 +wheel==0.46.3 +pip==26.0.1 +webencodings==0.5.1 +pure_eval==0.2.3 +ptyprocess==0.7.0 +nvidia-ml-py==13.590.48 +nvidia-cusparselt-cu12==0.7.1 +mpmath==1.3.0 +fastjsonschema==2.21.2 +zipp==3.23.0 +xyzservices==2025.11.0 +widgetsnbextension==4.0.15 +websocket-client==1.9.0 +webcolors==25.10.0 +wcwidth==0.6.0 +urllib3==2.6.3 +uri-template==1.3.0 +tzdata==2025.3 +typing_extensions==4.15.0 +triton==3.6.0 +traitlets==5.14.3 +tqdm==4.67.3 +tornado==6.5.5 +tinycss2==1.4.0 +sympy==1.14.0 +soupsieve==2.8.3 +smmap==5.0.3 +six==1.16.0 +sentencepiece==0.2.1 +Send2Trash==2.1.0 +safetensors==0.7.0 +rpds-py==0.30.0 +rfc3986-validator==0.1.1 +regex==2026.2.28 +pyzmq==27.1.0 +PyYAML==6.0.3 +python-json-logger==4.0.0 +Pygments==2.19.2 +pycparser==3.0 +psutil==7.2.2 +protobuf==4.24.4 +prometheus_client==0.24.1 +platformdirs==4.9.4 +pillow==11.3.0 +pexpect==4.9.0 +parso==0.8.6 +pandocfilters==1.5.1 +nvidia-nvtx-cu12==12.8.90 +nvidia-nvshmem-cu12==3.4.5 +nvidia-nvjitlink-cu12==12.8.93 +nvidia-nccl-cu12==2.27.5 +nvidia-curand-cu12==10.3.9.90 +nvidia-cufile-cu12==1.13.1.3 +nvidia-cuda-runtime-cu12==12.8.90 +nvidia-cuda-nvrtc-cu12==12.8.93 +nvidia-cuda-cupti-cu12==12.8.90 +nvidia-cublas-cu12==12.8.4.1 +numpy==1.26.4 +networkx==3.6.1 +nest-asyncio==1.6.0 +mistune==3.2.0 +MarkupSafe==3.0.3 +lark==1.3.1 +jupyterlab_widgets==3.0.16 +jupyterlab_pygments==0.3.0 +jsonpointer==3.0.0 +json5==0.13.0 +jmespath==1.1.0 +idna==3.11 +hf-xet==1.4.2 +h11==0.16.0 +fsspec==2026.2.0 +fqdn==1.5.1 +filelock==3.25.2 +executing==2.2.1 +einops==0.8.2 +defusedxml==0.7.1 +decorator==5.2.1 +debugpy==1.8.20 +cuda-pathfinder==1.4.2 +comm==0.2.3 +charset-normalizer==3.4.5 +certifi==2026.2.25 +bleach==6.3.0 +babel==2.18.0 +av==17.0.0 +attrs==25.4.0 +async-lru==2.2.0 +asttokens==3.0.1 +annotated-types==0.7.0 +typing-inspection==0.4.2 +terminado==0.18.1 +stack-data==0.6.3 +scipy==1.17.1 +rfc3987-syntax==1.1.0 +rfc3339-validator==0.1.4 +requests==2.32.5 +referencing==0.37.0 +python-dateutil==2.9.0.post0 +pydantic_core==2.41.5 +prompt_toolkit==3.0.52 +opencv-python-headless==4.11.0.86 +nvidia-cusparse-cu12==12.5.8.93 +nvidia-cufft-cu12==11.3.3.83 +nvidia-cudnn-cu12==9.10.2.21 +matplotlib-inline==0.2.1 +jupyter_core==5.9.1 +Jinja2==3.1.6 +jedi==0.19.2 +ipython_pygments_lexers==1.1.1 +importlib_metadata==8.7.1 +ImageIO==2.37.3 +httpcore==1.0.9 +gitdb==4.0.12 +cuda-bindings==12.9.4 +contourpy==1.3.3 +cffi==2.0.0 +beautifulsoup4==4.14.3 +anyio==4.12.1 +soundfile==0.13.1 +pydantic==2.12.5 +nvidia-cusolver-cu12==11.7.3.90 +jupyter_server_terminals==0.5.4 +jupyter_client==8.8.0 +jsonschema-specifications==2025.9.1 +ipython==9.11.0 +httpx==0.28.1 +GitPython==3.1.46 +botocore==1.42.68 +bokeh==3.9.0 +arrow==1.4.0 +argon2-cffi-bindings==25.1.0 +torch==2.10.0 +s3transfer==0.16.0 +jsonschema==4.26.0 +isoduration==20.11.0 +ipywidgets==8.1.8 +ipykernel==7.2.0 +argon2-cffi==25.1.0 +torchvision==0.25.0 +nbformat==5.10.4 +jupyter-console==6.6.3 +boto3==1.42.68 +accelerate==1.13.0 +nbclient==0.10.4 +jupyter-events==0.12.0 +nbconvert==7.17.0 +jupyter_server==2.17.0 +notebook_shim==0.2.4 +jupyterlab_server==2.28.0 +jupyter-lsp==2.3.0 +jupyterlab==4.5.6 +notebook==7.5.5 +jupyter==1.1.1 +fastgen==0.1.0 +pandas==3.0.1 +shellingham==1.5.4 +mdurl==0.1.2 +annotated-doc==0.0.4 +markdown-it-py==4.0.0 +rich==14.3.3 +typer==0.24.1 +huggingface_hub==1.7.1 +timm==1.0.25 +tokenizers==0.22.2 +diffusers==0.37.0 +transformers==5.3.0 +peft==0.18.1 +easydict==1.13 +lmdb==2.2.0 +threadpoolctl==3.6.0 +soxr==1.0.0 +msgpack==1.1.2 +llvmlite==0.47.0 +lazy-loader==0.5 +joblib==1.5.3 +audioread==3.1.0 +scikit-learn==1.8.0 +pooch==1.9.0 +numba==0.65.0 +librosa==0.11.0 +simsimd==6.5.16 +flatbuffers==25.12.19 +tifffile==2026.3.3 +stringzilla==4.6.0 +pyparsing==3.3.2 +prettytable==3.17.0 +onnx==1.17.0 +kiwisolver==1.5.0 +fonttools==4.62.1 +Cython==3.2.4 +cycler==0.12.1 +scikit-image==0.26.0 +onnxruntime==1.24.4 +matplotlib==3.10.8 +albucore==0.0.24 +albumentations==2.0.8 +insightface==0.7.3 +kornia_rs==0.1.10 +kornia==0.8.2 +pluggy==1.6.0 +iniconfig==2.3.0 +pytest==8.1.1 +dlib==20.0.1 +lpips==0.1.4 +python_speech_features==0.6 +decord==0.6.0 +click==8.2.1 +scenedetect==0.6.7.1 +torchaudio==2.10.0 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/wandb-metadata.json b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..ec4f02dfc61b2a7e81d5eb13d370af997d35708f --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/wandb-metadata.json @@ -0,0 +1,70 @@ +{ + "os": "Linux-5.15.0-151-generic-x86_64-with-glibc2.39", + "python": "CPython 3.12.12", + "startedAt": "2026-04-30T13:12:04.795735Z", + "args": [ + "--config=fastgen/configs/experiments/OmniAvatar/config_sf_full_ft_t769.py", + "-", + "trainer.resume=False", + "log_config.group=omniavatar_sf_full_ft", + "log_config.name=sf_full_ft_t769", + "log_config.project=OmniAvatar-FastGen", + "log_config.wandb_entity=paulhcho" + ], + "program": "/home/work/.local/hyunbin/FastGen-redmd/train.py", + "codePath": "train.py", + "codePathLocal": "train.py", + "git": { + "remote": "https://paulcho98:@github.com/paulcho98/FastGen.git", + "commit": "26958198402978118da258487f647a6a484d7548" + }, + "email": "paul.hyunbin@gmail.com", + "root": "/home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769", + "host": "main1", + "executable": "/home/work/.local/miniconda3/envs/hb_fastgen/bin/python3.12", + "cpu_count": 112, + "cpu_count_logical": 224, + "gpu": "NVIDIA H200", + "gpu_count": 4, + "disk": { + "/": { + "total": "1356758433792", + "used": "1159852929024" + } + }, + "memory": { + "total": "2163961778176" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-4685d4b3-5cf9-2766-43d3-b9615a684b7c" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ec888a66-4b6f-b8de-b34b-249efb9ad262" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-9c1e1773-d710-06c9-7db7-1b54e9fc3790" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2b1017dc-2958-a946-16d2-2c29da6d18b0" + } + ], + "cudaVersion": "12.9", + "writerId": "u9myhsec2v8a0ucd2m7c591akmnijuna" +} \ No newline at end of file diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/wandb-summary.json b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/logs/debug-core.log b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..68d6ad72097203849241a013c516a5a7bd3cd9f1 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/logs/debug-core.log @@ -0,0 +1,15 @@ +{"time":"2026-04-30T22:12:04.859299588+09:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpoa640hxo/port-1288706.txt","pid":1288706,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-04-30T22:12:04.859803722+09:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":1288706} +{"time":"2026-04-30T22:12:04.859791736+09:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1288706-1297267-3805102219/socket","Net":"unix"}} +{"time":"2026-04-30T22:12:05.04611902+09:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-04-30T22:12:05.061914302+09:00","level":"INFO","msg":"handleInformInit: received","streamId":"lvwg9pfc","id":"1(@)"} +{"time":"2026-04-30T22:12:05.464944536+09:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"lvwg9pfc","id":"1(@)"} +{"time":"2026-04-30T22:12:11.41991749+09:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"2wlt0s1ra7xt"} +{"time":"2026-05-01T10:11:37.611299577+09:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-05-01T10:11:37.61149483+09:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-05-01T10:11:37.611486469+09:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-05-01T10:11:37.611878894+09:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-05-01T10:11:37.611878464+09:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-1288706-1297267-3805102219/socket","Net":"unix"}} +{"time":"2026-05-01T10:11:39.633388942+09:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-05-01T10:11:39.633414388+09:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-05-01T10:11:39.633425107+09:00","level":"INFO","msg":"server is closed"} diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/logs/debug-internal.log b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..3e213ec5600b26d941826b41f0af25352b2c4da1 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/logs/debug-internal.log @@ -0,0 +1,15 @@ +{"time":"2026-04-30T22:12:05.062483501+09:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-04-30T22:12:05.463604741+09:00","level":"INFO","msg":"stream: created new stream","id":"lvwg9pfc"} +{"time":"2026-04-30T22:12:05.4636515+09:00","level":"INFO","msg":"handler: started","stream_id":"lvwg9pfc"} +{"time":"2026-04-30T22:12:05.464937177+09:00","level":"INFO","msg":"stream: started","id":"lvwg9pfc"} +{"time":"2026-04-30T22:12:05.464958887+09:00","level":"INFO","msg":"sender: started","stream_id":"lvwg9pfc"} +{"time":"2026-04-30T22:12:05.464963248+09:00","level":"INFO","msg":"writer: started","stream_id":"lvwg9pfc"} +{"time":"2026-05-01T10:11:37.611489893+09:00","level":"INFO","msg":"stream: closing","id":"lvwg9pfc"} +{"time":"2026-05-01T10:11:37.617805919+09:00","level":"ERROR","msg":"failed to write to chunk: failed to close: close /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/output.log: no space left on device"} +{"time":"2026-05-01T10:11:37.625534494+09:00","level":"ERROR","msg":"sender: failed to upload run summary: close /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/wandb-summary.json: no space left on device"} +{"time":"2026-05-01T10:11:37.635646188+09:00","level":"ERROR","msg":"sender: failed to upload run config: close /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/files/config.yaml: no space left on device"} +{"time":"2026-05-01T10:11:39.206389931+09:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-05-01T10:11:39.631573664+09:00","level":"INFO","msg":"handler: closed","stream_id":"lvwg9pfc"} +{"time":"2026-05-01T10:11:39.632071405+09:00","level":"ERROR","msg":"writer: failed closing store: transactionlog: error closing file: close /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/run-lvwg9pfc.wandb: no space left on device"} +{"time":"2026-05-01T10:11:39.633049656+09:00","level":"INFO","msg":"sender: closed","stream_id":"lvwg9pfc"} +{"time":"2026-05-01T10:11:39.633060653+09:00","level":"INFO","msg":"stream: closed","id":"lvwg9pfc"} diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/logs/debug.log b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..cdf1d978e77fdfa59e5c5b6bc2108ec5785bfe8e --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/logs/debug.log @@ -0,0 +1,21 @@ +2026-04-30 22:12:04,810 INFO MainThread:1288706 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-04-30 22:12:04,810 INFO MainThread:1288706 [wandb_setup.py:_flush():81] Configure stats pid to 1288706 +2026-04-30 22:12:04,810 INFO MainThread:1288706 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-04-30 22:12:04,810 INFO MainThread:1288706 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/logs/debug.log +2026-04-30 22:12:04,810 INFO MainThread:1288706 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/logs/debug-internal.log +2026-04-30 22:12:04,810 INFO MainThread:1288706 [wandb_init.py:init():844] calling init triggers +2026-04-30 22:12:04,810 INFO MainThread:1288706 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'log_config': {'project': 'OmniAvatar-FastGen', 'group': 'omniavatar_sf_full_ft', 'name': 'sf_full_ft_t769', 'wandb_mode': 'online', 'wandb_entity': 'paulhcho', 'wandb_credential': './credentials/wandb_api.txt'}, 'trainer': {'cudnn': {'deterministic': 'False', 'benchmark': 'True'}, 'checkpointer': {'save_dir': '/home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/checkpoints', 'use_s3': 'False', 's3_container': 's3://checkpoints/fastgen', 's3_credential': './credentials/s3.json', 'pretrained_ckpt_path': '/home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_t769_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth', 'pretrained_ckpt_key_map': {'net': 'net'}}, 'callbacks': {'grad_clip': {'grad_norm': '10.0', 'model_key': 'net', '_target_': ""}, 'gpu_stats': {'every_n': '100', '_target_': ""}, 'train_profiler': {'every_n': '100', '_target_': ""}, 'param_count': {'_target_': ""}, 'ema': {'type': 'constant', 'beta': '0.9999', 'gamma': '16.97', 'ema_halflife_kimg': '500', 'ema_rampup_ratio': '0.05', 'start_iter': '0', '_target_': ""}, 'wandb': {'sample_logging_iter': '100', '_target_': "", 'fps': '25', 'syncnet_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model'}}, 'save_ckpt_iter': '100', 'validation_iter': '100', 'skip_initial_validation': 'True', 'logging_iter': '1', 'max_iter': '5000', 'visualize_teacher': 'False', 'seed': '0', 'val_seed': None, 'resume': 'False', 'ddp': 'False', 'fsdp': 'True', 'tf32_enabled': 'True', 'grad_accum_rounds': '2', 'batch_size_global': None, 'offload_module_in_decoding': 'False', 'fsdp_cpu_offload': 'False', 'fsdp_min_num_params': '10000000', 'fsdp_sharding_group_size': None, 'global_vars': None, 'global_vars_val': [None], 'augment_pipe': None}, 'dataloader_train': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_path.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '8', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', '_target_': "", 'load_raw_audio': 'True', 'raw_audio_sample_rate': '16000', 'raw_audio_num_frames': '81', 'raw_audio_fps': '25.0'}, 'dataloader_val': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_val10.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '1', 'num_workers': '2', 'neg_text_emb_path': '/home/work/stableavatar_data/neg_text_emb.pt', 'use_ref_sequence': 'True', 'load_ode_path': 'False', '_target_': ''}, 'eval': {'num_samples': '50000', 'save_images': 'False', 'min_ckpt': '0', 'max_ckpt': '100000000', 'samples_dir': 'samples'}, 'model': {'net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'chunk_size': '3', 'total_num_frames': '21', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt', 'net_pred_type': 'flow', 'schedule_type': 'rf', 'local_attn_size': '7', 'sink_size': '1', 'use_dynamic_rope': 'True', '_target_': ""}, 'teacher': {'model_size': '14B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'fake_score_net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'guidance_scale': '4.5', 'timestep_cfg': {'enabled': 'True', 't_lo': '0.556', 't_hi': '0.882'}, 'skip_layers': None, 'net_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'net_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'sample_t_cfg': {'time_dist_type': 'shifted', 'train_p_mean': '-1.1', 'train_p_std': '2.0', 'shift': '5.0', 'min_t': '0.001', 'max_t': '0.999', 't_list': ['0.999', '0.769', '0.0'], 'log_t_df': '0.01'}, 'input_shape': ['16', '21', '64', '64'], 'device': 'cuda', 'grad_scaler_enabled': 'False', 'grad_scaler_init_scale': '65536.0', 'grad_scaler_growth_interval': '2000', 'pretrained_model_path': '', 'pretrained_student_net_path': '', 'load_student_weights': 'False', 'enable_preprocessors': 'True', 'use_ema': 'False', 'student_sample_steps': '2', 'student_sample_type': 'sde', 'fsdp_meta_init': 'False', 'add_teacher_to_fsdp_dict': 'True', 'ddp_find_unused_parameters': 'True', 'precision': 'bfloat16', 'precision_amp': None, 'precision_amp_infer': None, 'precision_amp_enc': None, 'precision_fsdp': 'float32', 'fake_score_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'fake_score_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'discriminator': {'feature_indices': '{0, 1, 2}', 'all_res': ['32', '16', '8'], 'in_channels': '256', '_target_': ""}, 'discriminator_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '0.0001', 'weight_decay': '0.01', 'betas': ['0.9', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'discriminator_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'student_update_freq': '5', 'gan_loss_weight_gen': '0', 'gan_use_same_t_noise': 'False', 'fake_score_pred_type': 'x0', 'gan_r1_reg_weight': '0.0', 'gan_r1_reg_alpha': '0.1', 'enable_gradient_in_rollout': 'True', 'start_gradient_frame': '0', 'same_step_across_blocks': 'True', 'last_step_only': 'False', 'context_noise': '0.0', 'fake_score': None, 'reward': {'enabled': 'True', 'checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model', 'input_fps': '25.0', 'audio_sample_rate': '16000', 'vshift': '15', 'decoder_kind': 'taew', 'taew_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth'}, 'reward_beta': '2', 'center_reward': 'False', 'clamp_reward': None, 'reward_weighting_mode': 'per_sample', 'save_reward_debug_video': 'False', 'reward_debug_dir': 'logs/redmd_debug'}, 'model_class': {'config': None, '_target_': ""}, '_wandb': {}} +2026-04-30 22:12:04,810 INFO MainThread:1288706 [wandb_init.py:init():892] starting backend +2026-04-30 22:12:05,046 INFO MainThread:1288706 [wandb_init.py:init():895] sending inform_init request +2026-04-30 22:12:05,059 INFO MainThread:1288706 [wandb_init.py:init():903] backend started and connected +2026-04-30 22:12:05,063 INFO MainThread:1288706 [wandb_init.py:init():973] updated telemetry +2026-04-30 22:12:05,082 INFO MainThread:1288706 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-04-30 22:12:06,169 INFO MainThread:1288706 [wandb_init.py:init():1042] starting run threads in backend +2026-04-30 22:12:06,416 INFO MainThread:1288706 [wandb_run.py:_console_start():2524] atexit reg +2026-04-30 22:12:06,416 INFO MainThread:1288706 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-04-30 22:12:06,416 INFO MainThread:1288706 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-04-30 22:12:06,416 INFO MainThread:1288706 [wandb_run.py:_redirect():2465] Redirects installed. +2026-04-30 22:12:06,420 INFO MainThread:1288706 [wandb_init.py:init():1082] run started, returning control to user process +2026-05-01 10:11:37,611 INFO wandb-AsyncioManager-main:1288706 [service_client.py:_forward_responses():134] Reached EOF. +2026-05-01 10:11:37,611 INFO wandb-AsyncioManager-main:1288706 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles. diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/run-lvwg9pfc.wandb b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/run-lvwg9pfc.wandb new file mode 100644 index 0000000000000000000000000000000000000000..5bd70c8bd45c095056172ca0992f4edd197630b1 --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb/run-20260430_221204-lvwg9pfc/run-lvwg9pfc.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f098cdaf0a019008f54a322101b0a7c6b098a58a0cb708a3301038db4f9d5ed8 +size 9895936 diff --git a/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb_id.txt b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb_id.txt new file mode 100644 index 0000000000000000000000000000000000000000..9b47f34743d410b917507f7832718802f87729cb --- /dev/null +++ b/hyunbin_FastGen-redmd/FASTGEN_OUTPUT_FULL_FT_T769/OmniAvatar-FastGen/omniavatar_sf_full_ft/sf_full_ft_t769/wandb_id.txt @@ -0,0 +1 @@ +lvwg9pfc