diff --git a/.gitattributes b/.gitattributes index b6fc3bc6f7d026c4ae9c1655b6dc42c34e1acfb7..866e7a9d7a8431f69158a939a934c1a610f11a3e 100644 --- a/.gitattributes +++ b/.gitattributes @@ -9973,3 +9973,207 @@ FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_NODFIN FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_NODFINIT/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_nodfinit/wandb/run-20260425_040609-23ktt1b1/files/media/videos/val0/reconstructed_300_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_NODFINIT/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_nodfinit/wandb/run-20260425_040609-23ktt1b1/files/media/videos/val0/reconstructed_300_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_LR3E6_NODFINIT/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_lr3e6_nodfinit/wandb/run-20260425_040609-23ktt1b1/run-23ktt1b1.wandb filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000100.fake_score_model/.metadata filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000100.fake_score_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000100.fake_score_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000100.fake_score_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000100.fake_score_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000100.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000100.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000100.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000100.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000100.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000200.fake_score_model/.metadata filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000200.fake_score_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000200.fake_score_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000200.fake_score_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000200.fake_score_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000200.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000200.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000200.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000200.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000200.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000300.fake_score_model/.metadata filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000300.fake_score_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000300.fake_score_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000300.fake_score_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000300.fake_score_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000300.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000300.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000300.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000300.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000300.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000400.fake_score_model/.metadata filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000400.fake_score_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000400.fake_score_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000400.fake_score_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000400.fake_score_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000400.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000400.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000400.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000400.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000400.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000500.fake_score_model/.metadata filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000500.fake_score_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000500.fake_score_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000500.fake_score_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000500.fake_score_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000500.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000500.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000500.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000500.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000500.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.fake_score_model/.metadata filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.fake_score_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.fake_score_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.fake_score_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.fake_score_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.fake_score_optim/.metadata filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.fake_score_optim/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.fake_score_optim/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.fake_score_optim/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.fake_score_optim/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.net_optim/.metadata filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.net_optim/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.net_optim/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.net_optim/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.net_optim/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260426_014355-3eo2d8ep/run-3eo2d8ep.wandb filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/train_media/data/real_100_47403c800591fce4a99b.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/train_media/data/real_200_4e05a85e239f00a00e38.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/train_media/data/real_300_97779d11e00d4fea84a0.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/train_media/data/real_400_112c0905448a821974f0.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/train_media/data/real_500_2594e32c120a620ede9d.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/train_media/data/real_600_aebf0f3e2314a893fcf4.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/train_media/student/generation_100_9ad898f857d040bf9b70.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/train_media/student/generation_200_ec698bf06b6b445b4e9e.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/train_media/student/generation_300_529218f9ad0b4522c2ed.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/train_media/student/generation_400_0da984eac02beebf8e3d.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/train_media/student/generation_500_d122305464d3ebb64647.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/train_media/student/generation_600_16faed3c3d83d799f457.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_100_0339428f7329a7be4e09.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_100_1df6e87dc4f30c2902b7.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_100_79c7e7fe822f87014a1f.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_100_8143de6481ab40f8b120.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_100_86cb2c4f20d248423637.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_100_9415105b25edacb0459a.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_100_bbdbcef0aac20e14f029.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_100_be9c98f50bcc5960eb6b.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_100_eebd5c16e611688d18b2.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_100_f77102c7b3d9fe62ef24.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_200_04334af08f58cf39aef4.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_200_3113f2a565b230be7f0e.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_200_4fa61c7c0dcdfbde8849.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_200_510833ef7d8e361e104d.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_200_516378fb4f514f2526a6.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_200_a2d95f36c14a032f6dbb.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_200_a91df4fe1abceb619c6a.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_200_ce354f7516e45328e3eb.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_200_da1eeeb78af6f345da3d.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_200_f5a4d6c3a812176408fa.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_300_1565131a5fdf095f0f38.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_300_30c8a4cb027f0417640c.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_300_3782221edf7d8217e421.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_300_4584f1b171cbf943b470.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_300_4d698752082729725cd8.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_300_60d9057ec37171fc7d5e.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_300_81c65606c72d6c7d8cc6.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_300_820ff8c5f6bb7ed3f6fb.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_300_874abc5f12e40930cdfd.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_300_e5472b72768a8bca34a0.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_400_0c07016fdc70f327cc1e.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_400_16406ece75a2e125e05b.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_400_36b0e00e14ad762f880f.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_400_4411bcf9d81d5de2a351.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_400_67e3449dc772bbbb398d.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_400_7743f87671b0c2293457.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_400_869fb4c31411e732bd65.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_400_9acc695f67b9d5a96303.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_400_c634c350c0c74c3af278.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_400_eeadc0c6b1921f934b7b.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_500_41f749a7b0dfc091bd0b.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_500_4e9bfad62f5eeb28a51d.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_500_556c9d12337a9883ffaa.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_500_628aa97b42a1200127a0.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_500_70bb27d15c124e9f0f2b.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_500_813f28f2e9c8ee3a8277.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_500_8b2afa111a3cb9c38644.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_500_d893a7c5a5d3bf77e987.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_500_d9aa9f2aa1576ba26432.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_500_fccb7da3bff4ab4d12ad.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_600_1467cb3b6f875c7e2639.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_600_3d60078af4607045df2f.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_600_43a0c6b539fdbd257093.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_600_43f30c27e3fa9e6c86ec.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_600_60df597251e026012d16.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_600_7b9509d46936637bcc68.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_600_82db3c09e17b84575be1.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_600_8ab6c8d2a459baf2075f.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_600_9225e2625894ebb01c82.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_600_aab249f793c15d258e05.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_100_15ac5b765a401d035db1.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_100_294040ac378ceaa34329.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_100_6ff046418b5d41477989.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_100_80aab00fcfa85d5db98e.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_100_884b2d197a5511393680.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_100_a076d17e202ed18ebdf8.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_100_c0196720a45889bb2198.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_100_c0f892b5fba78348af85.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_100_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_100_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_200_15ac5b765a401d035db1.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_200_294040ac378ceaa34329.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_200_6ff046418b5d41477989.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_200_80aab00fcfa85d5db98e.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_200_884b2d197a5511393680.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_200_a076d17e202ed18ebdf8.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_200_c0196720a45889bb2198.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_200_c0f892b5fba78348af85.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_200_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_200_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_300_15ac5b765a401d035db1.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_300_294040ac378ceaa34329.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_300_6ff046418b5d41477989.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_300_80aab00fcfa85d5db98e.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_300_884b2d197a5511393680.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_300_a076d17e202ed18ebdf8.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_300_c0196720a45889bb2198.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_300_c0f892b5fba78348af85.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_300_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_300_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_400_15ac5b765a401d035db1.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_400_294040ac378ceaa34329.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_400_6ff046418b5d41477989.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_400_80aab00fcfa85d5db98e.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_400_884b2d197a5511393680.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_400_a076d17e202ed18ebdf8.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_400_c0196720a45889bb2198.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_400_c0f892b5fba78348af85.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_400_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_400_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_500_15ac5b765a401d035db1.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_500_294040ac378ceaa34329.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_500_6ff046418b5d41477989.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_500_80aab00fcfa85d5db98e.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_500_884b2d197a5511393680.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_500_a076d17e202ed18ebdf8.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_500_c0196720a45889bb2198.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_500_c0f892b5fba78348af85.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_500_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_500_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_600_15ac5b765a401d035db1.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_600_294040ac378ceaa34329.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_600_6ff046418b5d41477989.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_600_80aab00fcfa85d5db98e.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_600_884b2d197a5511393680.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_600_a076d17e202ed18ebdf8.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_600_c0196720a45889bb2198.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_600_c0f892b5fba78348af85.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_600_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_600_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/run-yihddx5h.wandb filter=lfs diff=lfs merge=lfs -text diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000100.fake_score_model/.metadata b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000100.fake_score_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..051827b6d83c49e29da482575034a886aea64c29 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000100.fake_score_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fbf867948630105c68f9f08462557eee7adfb740e634b8405bf7afb70d33ec7d +size 1191995 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000100.fake_score_model/__0_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000100.fake_score_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..74794c41b279839d58b5607725c5963bf4f4decc --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000100.fake_score_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6b54231987c8f67d25974954d1c5fcf57a105d6dd4ecedac1fcf592d55b6537 +size 1599501209 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000100.fake_score_model/__1_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000100.fake_score_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..d03c8d4bbe8fb976b2d4030ef2581aa92bea7eed --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000100.fake_score_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f185571caa1a7b641316e276fc492d22b5cff3c8b8785abb5dec8830e3fb227 +size 1598334114 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000100.fake_score_model/__2_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000100.fake_score_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..1ad65b77632beb1919dfe3d54d2ee54ee07745fb --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000100.fake_score_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77f8a418d3ae9db2fe290ea93a497fdcb28f6378ed5f71eea1a08f560b020ae7 +size 1598358322 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000100.fake_score_model/__3_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000100.fake_score_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..1c74cd5c0ebbefd7a27a998e54210a8aff916b2b --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000100.fake_score_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:feec03351bfd48a004b497705a2fa52ea467ddd04e50dc4c064f9d68e04a8942 +size 1598356809 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000100.net_model/.metadata b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000100.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..d9e12c1b4bb7c99d99af3d66c074f36401da3649 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000100.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f013da828cf2be9b31cf2774ffc1ffb658f6892d0f12f46a20c48fde27218f29 +size 614432 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000100.net_model/__0_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000100.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..63e1771b2fcb6ef46ef75d23667bcad13e7dcdc7 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000100.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:acc57e6f8680262168d7839bcf0ee1f8d45fe60945552d8b0063ebe50aab5f10 +size 1424446005 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000100.net_model/__1_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000100.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..1b9e73ead19b057a27eaaf097f7660c480b92c9c --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000100.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42f5065e86614b000b448639e4d46145114bb23ec62ee6fedf857f9e2734fd74 +size 1401881227 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000100.net_model/__2_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000100.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..8e1b844791e4c5d87069ab7e67f134825ee9ec59 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000100.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:854b6f04220d6c3d2696e042e37bc03b25b872fc3c475ea5f4ca12c7c8edd668 +size 1406759189 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000100.net_model/__3_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000100.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..bcde31cd9c784df70212068af863b89351ed0191 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000100.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7548eae1bd31e1efeba6e00b0f838955013713321504053b9515abe5bd6fb036 +size 1401210558 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000100.pth b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000100.pth new file mode 100644 index 0000000000000000000000000000000000000000..8eba06d5534a7b2e93f2f5f3f62b7a9807edd0ae --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000100.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fefe455129d8d9683d7f02cdbb18043a7c8a6b6082c2b52130407de1795c467e +size 1901 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000200.fake_score_model/.metadata b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000200.fake_score_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..5d49559eaaa53bb0f401cd7126c13b82bc81bc92 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000200.fake_score_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c1f8f655dffc7ba715e61245114bc059b44fc6870bf4b544ad831090df707e9 +size 1191995 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000200.fake_score_model/__0_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000200.fake_score_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..31714af5c8701c6bd55e53558a4253fe00466658 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000200.fake_score_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99c8b1484a7c42d4e9d0bbf503e10ac63056245afe4d87c3416ba8ae19e8df1c +size 1599501209 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000200.fake_score_model/__1_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000200.fake_score_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..7678c607fe89e7ce5feecdba3906d907d36e846b --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000200.fake_score_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:290c72e94e9c1d810aa94f35c5a8ce5c2580b655299333b8adec791907415793 +size 1598334114 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000200.fake_score_model/__2_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000200.fake_score_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..bef7876b466b74a2333e940705946b9e8b7b37ee --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000200.fake_score_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a667eb87a27ba3a78d2e9a1127264b543eca42bb9d482d022e10c541faa10484 +size 1598358322 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000200.fake_score_model/__3_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000200.fake_score_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..7320eb60d97ba21e8f042aeac1d2cc741cffe7b4 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000200.fake_score_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c94b27c5362467befe37adcd6f6c9586030679cee77958c7bb015c64fdda9c7 +size 1598356809 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000200.net_model/.metadata b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000200.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..c6e4714ed9f046697426dc13dfe7df0a8fc41ec3 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000200.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e04bec9eb3bfae038902e9e43ebd2303297695f1c693be66c5b1579696df19b +size 614432 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000200.net_model/__0_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000200.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..776a66b272b3f7ee5a4ed67c0a59b598415016d4 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000200.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d33e755e4030ceae7de995ef337c71e6a7d2270774d1d52ac7aecf8e8fc16d0a +size 1424446005 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000200.net_model/__1_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000200.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..3ca3823fcbd404b3e0af4021d4d61c637c53ed68 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000200.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:edd7eb4d85363dde5f829741247f3078c739f167c58875a00e5c32c6ad29a361 +size 1401881227 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000200.net_model/__2_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000200.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..977d8bfbd89e0304974d3e9bf3cf12add33b3a37 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000200.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38da9a159961e15825156955dc16e117c54dadb5a6bb5bb6dcefbd88b3825096 +size 1406759189 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000200.net_model/__3_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000200.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..a7f904adf54a9724da637e2d96c883efeadbbd84 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000200.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e72408fdfe184ea1281f910fb276a8cc52a46e64eee9ae06b56d52baa28bd75d +size 1401210558 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000200.pth b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000200.pth new file mode 100644 index 0000000000000000000000000000000000000000..aad3029700e1ea1cb33285d1aa85bc71bb88a215 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000200.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ebfa62daa986c87b350c580ab69173f6912fff579c6397be2f5f01a2b5620cb +size 1901 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000300.fake_score_model/.metadata b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000300.fake_score_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..d0983ef4a415b501e72fc39ddf8d77ce63d182b2 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000300.fake_score_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:10aff434a9fc7d34385d1eb2e23b67d5509477b1d14c14ca4d0b5c199b403478 +size 1191995 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000300.fake_score_model/__0_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000300.fake_score_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..31b46293078d6eab9b07e60a88a4d6f334c0b79f --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000300.fake_score_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c0e5eb87e84f95004a0af46d764afe15f5f9507466f9fb2261e27854ddabcb9 +size 1599501209 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000300.fake_score_model/__1_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000300.fake_score_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..bc88143dd68475e9512a5dc4f8dd07a91146bbf8 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000300.fake_score_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0a449f7e053e9fdd1486184de6701f1e155ff088a769a524905a0c0b8078bb1 +size 1598334114 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000300.fake_score_model/__2_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000300.fake_score_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..15f3c45388062d617e802aa123f7b12213fdf199 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000300.fake_score_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74632c30240d2a479d00f0f32688c2a0b6f6cf0a7e478096bbce181457b0525d +size 1598358322 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000300.fake_score_model/__3_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000300.fake_score_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..748a4f4848765ecb028438a5ba08e22dc2d5ca57 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000300.fake_score_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81f80ef3d5bd56d6187688e89bd23b24b545564a5734fe758d0267cc561116ae +size 1598356809 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000300.net_model/.metadata b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000300.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..a0675690f34dd92f55317e4c4e97955398fb693d --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000300.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24c1be8c7c065c01f92dc5ced71208ace730e7f752e4a4d20a4b93b630528160 +size 614432 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000300.net_model/__0_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000300.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..0b922b053adc590ab2ab1535d2d6d9d880bd3670 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000300.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c341ddeb6a0334f2559402fb3bbabc51c8e1e5815d39fa4c7215438fb112d200 +size 1424446005 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000300.net_model/__1_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000300.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..fbcc6397838c4da1669b869e579e8c9ba456a54c --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000300.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e2fd271dd70b68134737cae24ae699b18408d207e74beff19a678b02833aca6 +size 1401881227 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000300.net_model/__2_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000300.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..04c7406cede259eb27832412ac9695dc5f12f7c3 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000300.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b9e86bc67054e40ec7d4e9c25c3dbd28a8e50d138db4afc6435c6dedea48a6e +size 1406759189 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000300.net_model/__3_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000300.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..4c0f73533b651c8fb875ea6a51e53957b5376a54 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000300.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07722efa5573ecdfdaa7a97604d7ebe822d2d97964267af9d62a37dc7b705280 +size 1401210558 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000300.pth b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000300.pth new file mode 100644 index 0000000000000000000000000000000000000000..92a76f0f3b040efee094daabdd04fc4ecb7c670e --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000300.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a76c71e02345bb8451425e4c637c86c4cc5f60c71d79309cf5524e46013f7295 +size 1901 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000400.fake_score_model/.metadata b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000400.fake_score_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..46e7fd8390a65457e26b3283207b234d5eadf422 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000400.fake_score_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8098d0d253da5d8f2f8dce8e7aada57396646745d1f631096f76ab5749a5cf8b +size 1191995 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000400.fake_score_model/__0_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000400.fake_score_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..28f56b1888896df5346dbb0b451f0eb38bd8fa67 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000400.fake_score_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6696d39dfb1d041ec8240b5a4b4e2b58e156fc8dc46b26aed8acbd2a2ef3caad +size 1599501209 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000400.fake_score_model/__1_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000400.fake_score_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..8aba230fabe5d2cddc2397df42d74d0df7a0aa2a --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000400.fake_score_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33c400aa8cb9af386b308bab9ef7643871939f543180d82b061c7df99e22fb4a +size 1598334114 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000400.fake_score_model/__2_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000400.fake_score_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..833335f853775b6c0d42eb0dfe2338e141c4ab51 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000400.fake_score_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:900a94bbc2662f2b75c4a8045fb64711c54fe8cf78bb2a5b4d37dd7b97d4030d +size 1598358322 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000400.fake_score_model/__3_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000400.fake_score_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..33060bb1a4487eea8496819235340580feb11260 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000400.fake_score_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34f3f28dbd0560e61337baa7246f37d45c356edc4283097c54ed462858a2f6e5 +size 1598356809 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000400.net_model/.metadata b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000400.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..4e8efe0e41e1b96f5e41f3f2f01cf763a17093d8 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000400.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d10ea16c1f3c84f8757d5abcd35eade39b8a7b81b98ae77c61c07ef72daea57 +size 614432 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000400.net_model/__0_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000400.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..57472b31cf25d40f0f31bf4ad7af6a694d4f3a8a --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000400.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:32b66966a723ae767f920b1e44590e16337fae80f7d3a50d0856bc04f38495b6 +size 1424446005 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000400.net_model/__1_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000400.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..be524b4f4b203dc0f3c39ffbbe8f3e9fcc48cba5 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000400.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d73e5e383a51ab9f8387f651bcd073d6b3b56d0187aaeef74f6eb4e407bbf6c +size 1401881227 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000400.net_model/__2_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000400.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..c6cb40e125ede14f395ff2238ea222b8fbaa96a3 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000400.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b318db145ec9e870d75a19d995fd92bd5080f61066f5988f59476fe85509817 +size 1406759189 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000400.net_model/__3_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000400.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..9bba7cda27256871b82e90c396edda1c6510a3fc --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000400.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84ca9b6b47158179e4a09e58ef5170398040d6daa15490384d75e73b004d6b31 +size 1401210558 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000400.pth b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000400.pth new file mode 100644 index 0000000000000000000000000000000000000000..6eed4d2ddbaaaa8d1ffb666f806efbca131032aa --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000400.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06ff7fb99b3daabe3678198e33994011726c066e03d570d82b78992320415261 +size 1901 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000500.fake_score_model/.metadata b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000500.fake_score_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..c46fa920fe96fdbff9eeb96be69707ea119d0017 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000500.fake_score_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f9e2a7e2da0c37562d1c81525d09e5be1b9574b0e9a74624b1c3ae75c8c7f6c +size 1191995 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000500.fake_score_model/__0_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000500.fake_score_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..89c1dca5e97a88a0bc325ab300260d6367beb34c --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000500.fake_score_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80dc7a20337faef3386f458d38e0dbb1cc665c78d4ef66452fea4e5ae058bd7b +size 1599501209 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000500.fake_score_model/__1_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000500.fake_score_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..e0d9434aa73614a65525dcc9778e334739c4fe1c --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000500.fake_score_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7da69aa310167a3abc3ef4323a09502c8f82502d03ad9ff35802f3067f41b0c3 +size 1598334114 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000500.fake_score_model/__2_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000500.fake_score_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..ffc4a705bc473132950c9b06e3cdae7100f7243c --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000500.fake_score_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1a4a05b64bdeb8a19c3fa10e4afee01c1fa5d02e435d34ca382d6bc8d59da17 +size 1598358322 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000500.fake_score_model/__3_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000500.fake_score_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..d1ab015adfd46081acbd7be4ef3147495a2dfb15 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000500.fake_score_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0a2e60db4394175648b2513082ccbf3b9d4246af347edec7d22703a644829cd +size 1598356809 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000500.net_model/.metadata b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000500.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..82f06286dfc5e5cd973ffa988a54439fa929f7cd --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000500.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a2c6ecc5ea212487598de07660632d912f7efda441e458aa54820ed329fb8c35 +size 614432 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000500.net_model/__0_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000500.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..91b27c09e28b3adee53fcb8691a8c3bcb6585274 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000500.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33127f2bc180f301b2a6a561bac4749ee19d510646c21eeacbb9b14b34bfecaa +size 1424446005 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000500.net_model/__1_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000500.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..bcf2c33bf3ea76e52264404e02bdd1eda4c862a1 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000500.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62fb6de5298def635ffb8fad8d180e6a796b609ed455cf9e1c2dbdefed0518f9 +size 1401881227 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000500.net_model/__2_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000500.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..2e698d9b2c35c29c17563b7b749849f30a22f3e4 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000500.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d0fec5fc339be76e18bf0dc09516f4b15c8831814342baed04e2aecef8dbce3 +size 1406759189 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000500.net_model/__3_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000500.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..6d49256c14ed4d5b2f28008874b4bf1221dc6a5b --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000500.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:32c1d7a60409dd15970f287af64fa3d1f2d89ba4557f203e7da61c1b4a7bab00 +size 1401210558 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000500.pth b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000500.pth new file mode 100644 index 0000000000000000000000000000000000000000..1eb73bc1d16a219be160dd7be09e2fe36228d59a --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000500.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0548d00c34d800d0bd0e3395f27252efde74712d987b8730e05e6bfaa7db4ff9 +size 1901 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.fake_score_model/.metadata b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.fake_score_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..7b1b681c9f63ef98b39cbd26a7f39268ee24894e --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.fake_score_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:861ed27bcc29fb1ba6036f31c4c3f4dc2ec4c7e3a4ae845161a8b5a8ff34c8ea +size 1191995 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.fake_score_model/__0_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.fake_score_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..e6d3cf9dd8b5119578fd6984b893c6b8cf7a7d5a --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.fake_score_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:112dbab0a3592b486f2c32a31300d9d581cb45dd0ba43113536c85eef0e3f787 +size 1599501209 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.fake_score_model/__1_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.fake_score_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..6540ce8561e84fc16d47e369d9d0cc2f61f53e19 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.fake_score_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51d84622a39978c22193693972cbdfe86031be20da7937d6185b8bb6a71e2795 +size 1598334114 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.fake_score_model/__2_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.fake_score_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..f90a69632fa102921d58498fd8192e807058e060 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.fake_score_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56e60c184cc4e98dd8da69f06da8d58216b1f955275f530fb35b5723619aabc8 +size 1598358322 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.fake_score_model/__3_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.fake_score_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..47f786193e7950307152a8cce0082accfeaa4191 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.fake_score_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df4e01ab8dab35fb54abcb5e2bf395fb1fad01792ca69c31cd80676155801bb4 +size 1598356809 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.fake_score_optim/.metadata b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.fake_score_optim/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..7593435d7e2b41e16212a9b3aaf2ffc46e487033 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.fake_score_optim/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bdd7931eff60eb775e227218bd4c7381f2fa370fe0fb930d9269e763e43a48f8 +size 1387279 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.fake_score_optim/__0_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.fake_score_optim/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..6f36ebd9faf05deac554de2a020b6d83aadbd5d1 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.fake_score_optim/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39d54cf5cdc4be3ca9bebf51bcfc3dad850fb28354739e2f53d54eab0c09d450 +size 351856217 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.fake_score_optim/__1_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.fake_score_optim/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..6112b36f93a559e05cb63b378cdc066b05ebe730 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.fake_score_optim/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3bdec03ef6ac3d7a8b8a32cf57a59a4d7473f2a9ab5363f8776ebec7b6d1c02f +size 352173823 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.fake_score_optim/__2_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.fake_score_optim/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..b739f22cb013acc109d1994e36b840edd882b031 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.fake_score_optim/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:abdea9e5be454ce8abc7490eb5ebc69b76688e35be31bf563d5079af9ffd448c +size 352173823 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.fake_score_optim/__3_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.fake_score_optim/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..ecbaac83809e0b7c8113d26d5e6489bb8c4c1d8b --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.fake_score_optim/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff88dcb742b04efd57a1d3845badb43370481a4614926e635b78c2c7686b8142 +size 352211338 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.net_model/.metadata b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..b9028a393a80ba4e63f2c8ee63f3d3c907238eb6 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f83dfefc76783d937d486c52a21345a12510f81fc42077b1e6bea4051824340 +size 614432 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.net_model/__0_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..d8c8ae7a19c218833fb05a0f1a4b94d1484e4fc4 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:73428e1b3e79fc95e791f8f9c0876b5f306ad8b70419107cecf451901d17f0da +size 1424446005 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.net_model/__1_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..6b8d13a8f36e0b0a84d91f43dc86a62b05f20b71 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d40efa35fd68d362defed7ead8cf572d336a1370d080c141e6639ff33d6a6ae +size 1401881227 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.net_model/__2_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..69ba25b5b9e678af9503713bcfcc602b30744a13 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63d50d1d9f829c782f272de6497b0684b506dd719222fe2304946eea1add6965 +size 1406759189 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.net_model/__3_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..18b6216d9bfd3fd064e42471ab40660769d42259 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:588ff2cbfdee888dd3813292535cc7777a18d422f80cecc9da7e349509bc68f8 +size 1401210558 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.net_optim/.metadata b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.net_optim/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..2767ad351126532f1548668f67fc463f2ae0b5ab --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.net_optim/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f4fed09100c7fd007506b354f774188abd18681d3f5b773135ef4e57b108341 +size 1649321 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.net_optim/__0_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.net_optim/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..bc5f1e66f74f4bf5dd50dcefc89de8ab585c8fc9 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.net_optim/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0abc81d8e2eea7fda56d907a93cdce96ea904df1b21b404630fde078c6494644 +size 2803269738 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.net_optim/__1_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.net_optim/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..ee30ee1f9fb2cc053f36a99158ec05f1dd24c185 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.net_optim/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f02bf24615afe39d9996f01495b55a06d86c039095073fadee845f33d24318d5 +size 2827716077 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.net_optim/__2_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.net_optim/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..2143f814881e0c1f14651961b70e7a0513ddd4df --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.net_optim/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd46c1c442844584727afa444ce624647347a5beb2b9a3e49995958639f75cb1 +size 2827716077 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.net_optim/__3_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.net_optim/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..0a86a2b7d130b55f288db7a44e33504f5dd27d87 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.net_optim/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eae136cad7c87ec4f351d6c2eb1e7d9faf5d74e11d2d57b0d43c646edfe00f4f +size 2811295404 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.pth b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.pth new file mode 100644 index 0000000000000000000000000000000000000000..09e6f0f7e9e1f82e7e414a183bb3d887608dcefe --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5700e1769b737e601814f3b65486731411f7425e226f38d9bde4ebfaaa855ba1 +size 1901 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/config.yaml b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..96833dec743834dac8fe085d20df017c1d1b4ac6 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/config.yaml @@ -0,0 +1,283 @@ +dataloader_train: + _target_: + batch_size: '8' + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_path.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_raw_audio: 'True' + neg_text_emb_path: null + num_workers: '2' + raw_audio_fps: '25.0' + raw_audio_num_frames: '81' + raw_audio_sample_rate: '16000' + use_ref_sequence: 'True' +dataloader_val: + _target_: + batch_size: '1' + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_val10.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_ode_path: 'False' + neg_text_emb_path: null + num_workers: '2' + use_ref_sequence: 'True' +eval: + max_ckpt: '100000000' + min_ckpt: '0' + num_samples: '50000' + samples_dir: samples + save_images: 'False' +log_config: + group: omniavatar_sf_audiofix + name: sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769 + project: OmniAvatar-FastGen + wandb_credential: ./credentials/wandb_api.txt + wandb_entity: paulhcho + wandb_mode: online +model: + add_teacher_to_fsdp_dict: 'True' + center_reward: 'False' + clamp_reward: null + context_noise: '0.0' + ddp_find_unused_parameters: 'True' + device: cuda + discriminator: + _target_: + all_res: + - '32' + - '16' + - '8' + feature_indices: '{0, 1, 2}' + in_channels: '256' + discriminator_optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'False' + lr: '0.0001' + model: null + optim_type: adamw + weight_decay: '0.01' + discriminator_scheduler: + _target_: + cycle_lengths: + - '10000000000' + f_max: + - '1.0' + f_min: + - '1.0' + f_start: + - 1e-06 + warm_up_steps: + - '0' + enable_gradient_in_rollout: 'True' + enable_preprocessors: 'True' + fake_score: null + fake_score_net: + _target_: + audio_hidden_size: '32' + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors + in_dim: '65' + merge_lora: 'False' + mode: v2v + model_size: 1.3B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt + schedule_type: rf + use_audio: 'True' + fake_score_optimizer: + _target_: + betas: + - '0.0' + - '0.999' + eps: 1e-08 + fused: 'False' + lr: 3e-06 + model: null + optim_type: adamw + weight_decay: '0.01' + fake_score_pred_type: x0 + fake_score_scheduler: + _target_: + cycle_lengths: + - '10000000000' + f_max: + - '1.0' + f_min: + - '1.0' + f_start: + - 1e-06 + warm_up_steps: + - '0' + fsdp_meta_init: 'False' + gan_loss_weight_gen: '0' + gan_r1_reg_alpha: '0.1' + gan_r1_reg_weight: '0.0' + gan_use_same_t_noise: 'False' + grad_scaler_enabled: 'False' + grad_scaler_growth_interval: '2000' + grad_scaler_init_scale: '65536.0' + guidance_scale: '4.5' + input_shape: + - '16' + - '21' + - '64' + - '64' + last_step_only: 'False' + load_student_weights: 'False' + net: + _target_: + audio_hidden_size: '32' + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors + chunk_size: '3' + in_dim: '65' + local_attn_size: '7' + mode: v2v + model_size: 1.3B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt + schedule_type: rf + sink_size: '1' + total_num_frames: '21' + use_audio: 'True' + use_dynamic_rope: 'True' + net_optimizer: + _target_: + betas: + - '0.0' + - '0.999' + eps: 1e-08 + fused: 'False' + lr: 2e-06 + model: null + optim_type: adamw + weight_decay: '0.01' + net_scheduler: + _target_: + cycle_lengths: + - '10000000000' + f_max: + - '1.0' + f_min: + - '1.0' + f_start: + - 1e-06 + warm_up_steps: + - '0' + precision: bfloat16 + precision_amp: null + precision_amp_enc: null + precision_amp_infer: null + precision_fsdp: float32 + pretrained_model_path: '' + pretrained_student_net_path: '' + reward: + audio_sample_rate: '16000' + checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model + decoder_kind: taew + enabled: 'True' + input_fps: '25.0' + taew_checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth + vshift: '15' + reward_beta: '2' + reward_debug_dir: logs/redmd_debug + reward_weighting_mode: per_sample + same_step_across_blocks: 'True' + sample_t_cfg: + log_t_df: '0.01' + max_t: '0.999' + min_t: '0.001' + shift: '5.0' + t_list: + - '0.999' + - '0.769' + - '0.0' + time_dist_type: shifted + train_p_mean: '-1.1' + train_p_std: '2.0' + save_reward_debug_video: 'False' + skip_layers: null + start_gradient_frame: '0' + student_sample_steps: '2' + student_sample_type: sde + student_update_freq: '5' + teacher: + _target_: + audio_hidden_size: '32' + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors + in_dim: '65' + merge_lora: 'True' + mode: v2v + model_size: 14B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt + schedule_type: rf + use_audio: 'True' + timestep_cfg: + enabled: 'True' + t_hi: '0.882' + t_lo: '0.556' + use_ema: 'False' +model_class: + _target_: + config: null +trainer: + augment_pipe: null + batch_size_global: null + callbacks: + ema: + _target_: + beta: '0.9999' + ema_halflife_kimg: '500' + ema_rampup_ratio: '0.05' + gamma: '16.97' + start_iter: '0' + type: constant + gpu_stats: + _target_: + every_n: '100' + grad_clip: + _target_: + grad_norm: '10.0' + model_key: net + param_count: + _target_: + train_profiler: + _target_: + every_n: '100' + wandb: + _target_: + fps: '25' + sample_logging_iter: '100' + syncnet_checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model + checkpointer: + pretrained_ckpt_key_map: + net: net + pretrained_ckpt_path: /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_t769_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth + s3_container: s3://checkpoints/fastgen + s3_credential: ./credentials/s3.json + save_dir: /tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints + use_s3: 'False' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: 'False' + fsdp: 'True' + fsdp_cpu_offload: 'False' + fsdp_min_num_params: '10000000' + fsdp_sharding_group_size: null + global_vars: null + global_vars_val: + - null + grad_accum_rounds: '2' + logging_iter: '1' + max_iter: '5000' + offload_module_in_decoding: 'False' + resume: 'False' + save_ckpt_iter: '100' + seed: '0' + skip_initial_validation: 'True' + tf32_enabled: 'True' + val_seed: null + validation_iter: '100' + visualize_teacher: 'False' diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/debug-internal.log b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..d176778b9d99660ce81e2183c6c8f0534344eb33 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/debug-internal.log @@ -0,0 +1,6 @@ +{"time":"2026-04-27T10:02:46.236048851+09:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-04-27T10:02:46.734345704+09:00","level":"INFO","msg":"stream: created new stream","id":"yihddx5h"} +{"time":"2026-04-27T10:02:46.73501439+09:00","level":"INFO","msg":"handler: started","stream_id":"yihddx5h"} +{"time":"2026-04-27T10:02:46.735085357+09:00","level":"INFO","msg":"stream: started","id":"yihddx5h"} +{"time":"2026-04-27T10:02:46.735100956+09:00","level":"INFO","msg":"sender: started","stream_id":"yihddx5h"} +{"time":"2026-04-27T10:02:46.735126398+09:00","level":"INFO","msg":"writer: started","stream_id":"yihddx5h"} diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/debug.log b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..82089e6d2aa3595f58dcf5af0e4e2ca3a4a87edb --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/debug.log @@ -0,0 +1,19 @@ +2026-04-27 10:02:45,746 INFO MainThread:1078060 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-04-27 10:02:45,746 INFO MainThread:1078060 [wandb_setup.py:_flush():81] Configure stats pid to 1078060 +2026-04-27 10:02:45,747 INFO MainThread:1078060 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-04-27 10:02:45,747 INFO MainThread:1078060 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/logs/debug.log +2026-04-27 10:02:45,747 INFO MainThread:1078060 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/logs/debug-internal.log +2026-04-27 10:02:45,747 INFO MainThread:1078060 [wandb_init.py:init():844] calling init triggers +2026-04-27 10:02:45,747 INFO MainThread:1078060 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'log_config': {'project': 'OmniAvatar-FastGen', 'group': 'omniavatar_sf_audiofix', 'name': 'sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769', 'wandb_mode': 'online', 'wandb_entity': 'paulhcho', 'wandb_credential': './credentials/wandb_api.txt'}, 'trainer': {'cudnn': {'deterministic': 'False', 'benchmark': 'True'}, 'checkpointer': {'save_dir': '/tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints', 'use_s3': 'False', 's3_container': 's3://checkpoints/fastgen', 's3_credential': './credentials/s3.json', 'pretrained_ckpt_path': '/home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_t769_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth', 'pretrained_ckpt_key_map': {'net': 'net'}}, 'callbacks': {'grad_clip': {'grad_norm': '10.0', 'model_key': 'net', '_target_': ""}, 'gpu_stats': {'every_n': '100', '_target_': ""}, 'train_profiler': {'every_n': '100', '_target_': ""}, 'param_count': {'_target_': ""}, 'ema': {'type': 'constant', 'beta': '0.9999', 'gamma': '16.97', 'ema_halflife_kimg': '500', 'ema_rampup_ratio': '0.05', 'start_iter': '0', '_target_': ""}, 'wandb': {'sample_logging_iter': '100', '_target_': "", 'fps': '25', 'syncnet_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model'}}, 'save_ckpt_iter': '100', 'validation_iter': '100', 'skip_initial_validation': 'True', 'logging_iter': '1', 'max_iter': '5000', 'visualize_teacher': 'False', 'seed': '0', 'val_seed': None, 'resume': 'False', 'ddp': 'False', 'fsdp': 'True', 'tf32_enabled': 'True', 'grad_accum_rounds': '2', 'batch_size_global': None, 'offload_module_in_decoding': 'False', 'fsdp_cpu_offload': 'False', 'fsdp_min_num_params': '10000000', 'fsdp_sharding_group_size': None, 'global_vars': None, 'global_vars_val': [None], 'augment_pipe': None}, 'dataloader_train': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_path.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '8', 'num_workers': '2', 'neg_text_emb_path': None, 'use_ref_sequence': 'True', '_target_': "", 'load_raw_audio': 'True', 'raw_audio_sample_rate': '16000', 'raw_audio_num_frames': '81', 'raw_audio_fps': '25.0'}, 'dataloader_val': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_val10.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '1', 'num_workers': '2', 'neg_text_emb_path': None, 'use_ref_sequence': 'True', 'load_ode_path': 'False', '_target_': ''}, 'eval': {'num_samples': '50000', 'save_images': 'False', 'min_ckpt': '0', 'max_ckpt': '100000000', 'samples_dir': 'samples'}, 'model': {'net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'chunk_size': '3', 'total_num_frames': '21', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt', 'net_pred_type': 'flow', 'schedule_type': 'rf', 'local_attn_size': '7', 'sink_size': '1', 'use_dynamic_rope': 'True', '_target_': ""}, 'teacher': {'model_size': '14B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'fake_score_net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt', 'merge_lora': 'False', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'guidance_scale': '4.5', 'timestep_cfg': {'enabled': 'True', 't_lo': '0.556', 't_hi': '0.882'}, 'skip_layers': None, 'net_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'net_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'sample_t_cfg': {'time_dist_type': 'shifted', 'train_p_mean': '-1.1', 'train_p_std': '2.0', 'shift': '5.0', 'min_t': '0.001', 'max_t': '0.999', 't_list': ['0.999', '0.769', '0.0'], 'log_t_df': '0.01'}, 'input_shape': ['16', '21', '64', '64'], 'device': 'cuda', 'grad_scaler_enabled': 'False', 'grad_scaler_init_scale': '65536.0', 'grad_scaler_growth_interval': '2000', 'pretrained_model_path': '', 'pretrained_student_net_path': '', 'load_student_weights': 'False', 'enable_preprocessors': 'True', 'use_ema': 'False', 'student_sample_steps': '2', 'student_sample_type': 'sde', 'fsdp_meta_init': 'False', 'add_teacher_to_fsdp_dict': 'True', 'ddp_find_unused_parameters': 'True', 'precision': 'bfloat16', 'precision_amp': None, 'precision_amp_infer': None, 'precision_amp_enc': None, 'precision_fsdp': 'float32', 'fake_score_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '3e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'fake_score_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'discriminator': {'feature_indices': '{0, 1, 2}', 'all_res': ['32', '16', '8'], 'in_channels': '256', '_target_': ""}, 'discriminator_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '0.0001', 'weight_decay': '0.01', 'betas': ['0.9', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'discriminator_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'student_update_freq': '5', 'gan_loss_weight_gen': '0', 'gan_use_same_t_noise': 'False', 'fake_score_pred_type': 'x0', 'gan_r1_reg_weight': '0.0', 'gan_r1_reg_alpha': '0.1', 'enable_gradient_in_rollout': 'True', 'start_gradient_frame': '0', 'same_step_across_blocks': 'True', 'last_step_only': 'False', 'context_noise': '0.0', 'fake_score': None, 'reward': {'enabled': 'True', 'checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model', 'input_fps': '25.0', 'audio_sample_rate': '16000', 'vshift': '15', 'decoder_kind': 'taew', 'taew_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth'}, 'reward_beta': '2', 'center_reward': 'False', 'clamp_reward': None, 'reward_weighting_mode': 'per_sample', 'save_reward_debug_video': 'False', 'reward_debug_dir': 'logs/redmd_debug'}, 'model_class': {'config': None, '_target_': ""}, '_wandb': {}} +2026-04-27 10:02:45,747 INFO MainThread:1078060 [wandb_init.py:init():892] starting backend +2026-04-27 10:02:46,216 INFO MainThread:1078060 [wandb_init.py:init():895] sending inform_init request +2026-04-27 10:02:46,231 INFO MainThread:1078060 [wandb_init.py:init():903] backend started and connected +2026-04-27 10:02:46,235 INFO MainThread:1078060 [wandb_init.py:init():973] updated telemetry +2026-04-27 10:02:46,265 INFO MainThread:1078060 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-04-27 10:02:47,391 INFO MainThread:1078060 [wandb_init.py:init():1042] starting run threads in backend +2026-04-27 10:02:47,739 INFO MainThread:1078060 [wandb_run.py:_console_start():2524] atexit reg +2026-04-27 10:02:47,739 INFO MainThread:1078060 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-04-27 10:02:47,739 INFO MainThread:1078060 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-04-27 10:02:47,739 INFO MainThread:1078060 [wandb_run.py:_redirect():2465] Redirects installed. +2026-04-27 10:02:47,746 INFO MainThread:1078060 [wandb_init.py:init():1082] run started, returning control to user process diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260426_014355-3eo2d8ep/files/config.yaml b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260426_014355-3eo2d8ep/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..63083992e19ec3bc50e58f75a0854fc72ba64eb4 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260426_014355-3eo2d8ep/files/config.yaml @@ -0,0 +1,389 @@ +_wandb: + value: + cli_version: 0.25.0 + e: + upk6yt8d6gxnny9vydatyl2lnqzs7oye: + args: + - --config=fastgen/configs/experiments/OmniAvatar/config_sf_sink1_window7_redmd_beta2_taew.py + - '-' + - trainer.resume=False + - model.fake_score_optimizer.lr=3e-6 + - log_config.group=omniavatar_sf_audiofix + - log_config.name=sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769 + - log_config.project=OmniAvatar-FastGen + - log_config.wandb_entity=paulhcho + - model.sample_t_cfg.t_list=[0.999,0.769,0.0] + codePath: train.py + codePathLocal: train.py + cpu_count: 112 + cpu_count_logical: 224 + cudaVersion: "12.9" + disk: + /: + total: "1356758433792" + used: "979257122816" + email: paul.hyunbin@gmail.com + executable: /home/work/.local/miniconda3/envs/hb_fastgen/bin/python3.12 + git: + commit: 302cf2d9ddcceacc8fca5e480cb21b26ea6c07be + remote: https://paulcho98:@github.com/paulcho98/FastGen.git + gpu: NVIDIA H200 + gpu_count: 4 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-4685d4b3-5cf9-2766-43d3-b9615a684b7c + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-ec888a66-4b6f-b8de-b34b-249efb9ad262 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-9c1e1773-d710-06c9-7db7-1b54e9fc3790 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-2b1017dc-2958-a946-16d2-2c29da6d18b0 + host: main1 + memory: + total: "2163961778176" + os: Linux-5.15.0-151-generic-x86_64-with-glibc2.39 + program: /home/work/.local/hyunbin/FastGen-redmd/train.py + python: CPython 3.12.12 + root: /tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769 + startedAt: "2026-04-25T16:43:55.700882Z" + writerId: upk6yt8d6gxnny9vydatyl2lnqzs7oye + m: [] + python_version: 3.12.12 + t: + "1": + - 1 + - 5 + - 11 + - 41 + - 49 + - 50 + - 53 + - 63 + - 71 + - 83 + - 98 + "2": + - 1 + - 5 + - 11 + - 41 + - 49 + - 50 + - 53 + - 63 + - 71 + - 83 + - 98 + "3": + - 13 + - 14 + - 16 + - 61 + - 62 + "4": 3.12.12 + "5": 0.25.0 + "6": 5.3.0 + "12": 0.25.0 + "13": linux-x86_64 +dataloader_train: + value: + _target_: + batch_size: "8" + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_path.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_raw_audio: "True" + neg_text_emb_path: null + num_workers: "2" + raw_audio_fps: "25.0" + raw_audio_num_frames: "81" + raw_audio_sample_rate: "16000" + use_ref_sequence: "True" +dataloader_val: + value: + _target_: + batch_size: "1" + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_val10.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_ode_path: "False" + neg_text_emb_path: null + num_workers: "2" + use_ref_sequence: "True" +eval: + value: + max_ckpt: "100000000" + min_ckpt: "0" + num_samples: "50000" + samples_dir: samples + save_images: "False" +log_config: + value: + group: omniavatar_sf_audiofix + name: sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769 + project: OmniAvatar-FastGen + wandb_credential: ./credentials/wandb_api.txt + wandb_entity: paulhcho + wandb_mode: online +model: + value: + add_teacher_to_fsdp_dict: "True" + center_reward: "False" + clamp_reward: null + context_noise: "0.0" + ddp_find_unused_parameters: "True" + device: cuda + discriminator: + _target_: + all_res: + - "32" + - "16" + - "8" + feature_indices: '{0, 1, 2}' + in_channels: "256" + discriminator_optimizer: + _target_: + betas: + - "0.9" + - "0.999" + eps: "1e-08" + fused: "False" + lr: "0.0001" + model: null + optim_type: adamw + weight_decay: "0.01" + discriminator_scheduler: + _target_: + cycle_lengths: + - "10000000000" + f_max: + - "1.0" + f_min: + - "1.0" + f_start: + - "1e-06" + warm_up_steps: + - "0" + enable_gradient_in_rollout: "True" + enable_preprocessors: "True" + fake_score: null + fake_score_net: + _target_: + audio_hidden_size: "32" + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors + in_dim: "65" + merge_lora: "False" + mode: v2v + model_size: 1.3B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt + schedule_type: rf + use_audio: "True" + fake_score_optimizer: + _target_: + betas: + - "0.0" + - "0.999" + eps: "1e-08" + fused: "False" + lr: "3e-06" + model: null + optim_type: adamw + weight_decay: "0.01" + fake_score_pred_type: x0 + fake_score_scheduler: + _target_: + cycle_lengths: + - "10000000000" + f_max: + - "1.0" + f_min: + - "1.0" + f_start: + - "1e-06" + warm_up_steps: + - "0" + fsdp_meta_init: "False" + gan_loss_weight_gen: "0" + gan_r1_reg_alpha: "0.1" + gan_r1_reg_weight: "0.0" + gan_use_same_t_noise: "False" + grad_scaler_enabled: "False" + grad_scaler_growth_interval: "2000" + grad_scaler_init_scale: "65536.0" + guidance_scale: "4.5" + input_shape: + - "16" + - "21" + - "64" + - "64" + last_step_only: "False" + load_student_weights: "False" + net: + _target_: + audio_hidden_size: "32" + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors + chunk_size: "3" + in_dim: "65" + local_attn_size: "7" + mode: v2v + model_size: 1.3B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt + schedule_type: rf + sink_size: "1" + total_num_frames: "21" + use_audio: "True" + use_dynamic_rope: "True" + net_optimizer: + _target_: + betas: + - "0.0" + - "0.999" + eps: "1e-08" + fused: "False" + lr: "2e-06" + model: null + optim_type: adamw + weight_decay: "0.01" + net_scheduler: + _target_: + cycle_lengths: + - "10000000000" + f_max: + - "1.0" + f_min: + - "1.0" + f_start: + - "1e-06" + warm_up_steps: + - "0" + precision: bfloat16 + precision_amp: null + precision_amp_enc: null + precision_amp_infer: null + precision_fsdp: float32 + pretrained_model_path: "" + pretrained_student_net_path: "" + reward: + audio_sample_rate: "16000" + checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model + decoder_kind: taew + enabled: "True" + input_fps: "25.0" + taew_checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth + vshift: "15" + reward_beta: "2" + reward_debug_dir: logs/redmd_debug + reward_weighting_mode: per_sample + same_step_across_blocks: "True" + sample_t_cfg: + log_t_df: "0.01" + max_t: "0.999" + min_t: "0.001" + shift: "5.0" + t_list: + - "0.999" + - "0.769" + - "0.0" + time_dist_type: shifted + train_p_mean: "-1.1" + train_p_std: "2.0" + save_reward_debug_video: "False" + skip_layers: null + start_gradient_frame: "0" + student_sample_steps: "2" + student_sample_type: sde + student_update_freq: "5" + teacher: + _target_: + audio_hidden_size: "32" + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors + in_dim: "65" + merge_lora: "True" + mode: v2v + model_size: 14B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt + schedule_type: rf + use_audio: "True" + timestep_cfg: + enabled: "True" + t_hi: "0.882" + t_lo: "0.556" + use_ema: "False" +model_class: + value: + _target_: + config: null +trainer: + value: + augment_pipe: null + batch_size_global: null + callbacks: + ema: + _target_: + beta: "0.9999" + ema_halflife_kimg: "500" + ema_rampup_ratio: "0.05" + gamma: "16.97" + start_iter: "0" + type: constant + gpu_stats: + _target_: + every_n: "100" + grad_clip: + _target_: + grad_norm: "10.0" + model_key: net + param_count: + _target_: + train_profiler: + _target_: + every_n: "100" + wandb: + _target_: + fps: "25" + sample_logging_iter: "100" + syncnet_checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model + checkpointer: + pretrained_ckpt_key_map: + net: net + pretrained_ckpt_path: /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth + s3_container: s3://checkpoints/fastgen + s3_credential: ./credentials/s3.json + save_dir: /tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints + use_s3: "False" + cudnn: + benchmark: "True" + deterministic: "False" + ddp: "False" + fsdp: "True" + fsdp_cpu_offload: "False" + fsdp_min_num_params: "10000000" + fsdp_sharding_group_size: null + global_vars: null + global_vars_val: + - null + grad_accum_rounds: "2" + logging_iter: "1" + max_iter: "5000" + offload_module_in_decoding: "False" + resume: "False" + save_ckpt_iter: "100" + seed: "0" + skip_initial_validation: "True" + tf32_enabled: "True" + val_seed: null + validation_iter: "100" + visualize_teacher: "False" diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260426_014355-3eo2d8ep/files/output.log b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260426_014355-3eo2d8ep/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..0a13bdeeaaa870ab5ab989c37388a3888a840dcc --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260426_014355-3eo2d8ep/files/output.log @@ -0,0 +1,215 @@ +[Apr 26, 2026 - 01:43:57 | INFO | fastgen.callbacks.wandb:init_wandb:290 ] Wandb code upload disabled (set WANDB_UPLOAD_CODE=true to enable) +[Apr 26, 2026 - 01:43:57 | SUCCESS | fastgen.trainer:__init__:53 ] Callbacks initialized successfully +[Apr 26, 2026 - 01:43:57 | INFO | fastgen.trainer:__init__:57 ] Callback synchronization complete +[Apr 26, 2026 - 01:43:57 | INFO | fastgen.trainer:__init__:60 ] Initializing checkpointer... +[Apr 26, 2026 - 01:43:57 | SUCCESS | fastgen.trainer:__init__:65 ] Checkpointer initialized successfully +[Apr 26, 2026 - 01:43:57 | SUCCESS | __main__:main:33 ] Trainer initialized successfully +[Apr 26, 2026 - 01:43:57 | INFO | fastgen.trainer:run:77 ] Starting training +[Apr 26, 2026 - 01:43:57 | INFO | fastgen.trainer:run:80 ] Initializing callbacks and model ... +[Apr 26, 2026 - 01:43:57 | INFO | fastgen.utils.checkpointer:load:151 ] Loading model from /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth +[Apr 26, 2026 - 01:44:22 | INFO | fastgen.utils.checkpointer:load:154 ] Loading the model_dict... +[Apr 26, 2026 - 01:44:34 | INFO | fastgen.utils.checkpointer:load:159 ] Model net, loading info: +[Apr 26, 2026 - 01:44:34 | INFO | fastgen.trainer:load_pretrained_ckpt:252 ] Loaded net model from net in /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth at iteration 5000 +[Apr 26, 2026 - 01:44:34 | INFO | fastgen.trainer:load_pretrained_ckpt:262 ] Setting resume_iter for model to 5000. +[Apr 26, 2026 - 01:44:34 | INFO | fastgen.trainer:run:95 ] Starting model.on_train_begin ... +[Apr 26, 2026 - 01:45:40 | INFO | fastgen.methods.model:on_train_begin:296 ] Teacher check: add_teacher_to_fsdp_dict=True, fsdp_dict keys=['net', 'fake_score', 'teacher'], teacher in fsdp_dict=True +[Apr 26, 2026 - 01:45:40 | INFO | fastgen.trainer:run:99 ] model.on_train_begin completed +[Apr 26, 2026 - 01:45:40 | INFO | fastgen.trainer:run:110 ] Wrapping model into fsdp .. +[Apr 26, 2026 - 01:45:40 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:102 ] Fully sharding model with 4 ranks... +[Apr 26, 2026 - 01:45:40 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'net' (1.42B params)... +[Apr 26, 2026 - 01:45:40 | INFO | fastgen.networks.OmniAvatar.network_causal:fully_shard:2166 ] CausalOmniAvatarWan: keeping manual gradient checkpointing (not using apply_fsdp_checkpointing due to KV cache dynamics) +[Apr 26, 2026 - 01:45:43 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[Apr 26, 2026 - 01:45:43 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped net in 3.0s +[Apr 26, 2026 - 01:45:43 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'fake_score' (1.60B params)... +[Apr 26, 2026 - 01:45:43 | INFO | fastgen.networks.OmniAvatar.network:fully_shard:774 ] OmniAvatarWan: keeping manual gradient checkpointing (checkpoint_wrapper incompatible with inter-block audio injection) +[Apr 26, 2026 - 01:45:45 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[Apr 26, 2026 - 01:45:46 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped fake_score in 2.8s +[Apr 26, 2026 - 01:45:46 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'teacher' (14.29B params)... +[Apr 26, 2026 - 01:45:46 | INFO | fastgen.networks.OmniAvatar.network:fully_shard:774 ] OmniAvatarWan: keeping manual gradient checkpointing (checkpoint_wrapper incompatible with inter-block audio injection) +[Apr 26, 2026 - 01:45:58 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[Apr 26, 2026 - 01:45:59 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped teacher in 13.2s +[Apr 26, 2026 - 01:45:59 | INFO | fastgen.trainer:run:118 ] FSDP wrapping completed +[Apr 26, 2026 - 01:45:59 | INFO | fastgen.callbacks.ema:on_model_init_end:64 ] EMA ema is not enabled, skipping callback. +[Apr 26, 2026 - 01:45:59 | INFO | fastgen.trainer:run:133 ] Auto-Resume Details: None +[Apr 26, 2026 - 01:45:59 | INFO | fastgen.utils.basic_utils:set_random_seed:144 ] Using random seed 0. +[Apr 26, 2026 - 01:45:59 | INFO | fastgen.trainer:run:165 ] Instantiating dataloader... +[Apr 26, 2026 - 01:46:05 | INFO | fastgen.callbacks.wandb:on_dataloader_init_end:422 ] SKIP_GT_VAL_UPLOAD=1 — skipping GT val video upload +[Apr 26, 2026 - 01:46:05 | INFO | fastgen.callbacks.gpu_stats:on_train_begin:57 ] every_n to measure gpus stats: 1 +[Apr 26, 2026 - 01:46:05 | INFO | fastgen.callbacks.train_profiler:on_train_begin:54 ] every_n to profile trainer: 1 +[Apr 26, 2026 - 01:46:05 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] model (OmniAvatarSelfForcingReDMD) has 1596.36 M trainable and 17325.46 M total params (logical). +[Apr 26, 2026 - 01:46:05 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 420.42 M trainable and 4364.06 M total params LOCAL on rank 0. +[Apr 26, 2026 - 01:46:05 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 420.14 M trainable and 4362.26 M total params LOCAL on rank 1. +[Apr 26, 2026 - 01:46:05 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 420.14 M trainable and 4362.26 M total params LOCAL on rank 2. +[Apr 26, 2026 - 01:46:05 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 420.14 M trainable and 4362.26 M total params LOCAL on rank 3. +[Apr 26, 2026 - 01:46:05 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] net (CausalOmniAvatarWan) has 1421.38 M trainable and 1421.38 M total params (logical). +[Apr 26, 2026 - 01:46:05 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 376.68 M trainable and 376.68 M total params LOCAL on rank 0. +[Apr 26, 2026 - 01:46:05 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 376.40 M trainable and 376.40 M total params LOCAL on rank 1. +[Apr 26, 2026 - 01:46:05 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 376.40 M trainable and 376.40 M total params LOCAL on rank 2. +[Apr 26, 2026 - 01:46:05 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 376.40 M trainable and 376.40 M total params LOCAL on rank 3. +[Apr 26, 2026 - 01:46:05 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] fake_score (OmniAvatarWan) has 174.98 M trainable and 1596.36 M total params (logical). +[Apr 26, 2026 - 01:46:05 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 43.75 M trainable and 399.30 M total params LOCAL on rank 0. +[Apr 26, 2026 - 01:46:05 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 43.75 M trainable and 399.02 M total params LOCAL on rank 1. +[Apr 26, 2026 - 01:46:05 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 43.75 M trainable and 399.02 M total params LOCAL on rank 2. +[Apr 26, 2026 - 01:46:05 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 43.75 M trainable and 399.02 M total params LOCAL on rank 3. +[Apr 26, 2026 - 01:46:05 | INFO | fastgen.trainer:run:174 ] iter_start: 0 +[Apr 26, 2026 - 01:46:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.591722 38.636120 38.540211 +peak_gpu_mem_gb 60.567137 60.567137 60.567137 +peak_gpu_mem_reserved_gb 62.595703 62.595703 62.595703 +util 90.500000 95.000000 83.000000 +[Apr 26, 2026 - 01:46:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 1 : data loading time 1.29 +[Apr 26, 2026 - 01:46:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 1 : avg forward pass time 10.68 +[Apr 26, 2026 - 01:46:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 1 : backward pass time 11.57 +[Apr 26, 2026 - 01:46:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 1 : optimizer step time 1.06 +[Apr 26, 2026 - 01:46:54 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 1-------------------- +[Apr 26, 2026 - 01:46:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0084 iter count: 1.0 +[Apr 26, 2026 - 01:46:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[Apr 26, 2026 - 01:46:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 01:46:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 01:47:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.767372 38.812073 38.715778 +peak_gpu_mem_gb 62.354346 62.354346 62.354346 +peak_gpu_mem_reserved_gb 64.042969 64.042969 64.042969 +util 97.500000 99.000000 93.000000 +[Apr 26, 2026 - 01:47:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 2 : avg iteration time 48.82 seconds +[Apr 26, 2026 - 01:47:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 2 : data loading time 0.00 +[Apr 26, 2026 - 01:47:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 2 : avg forward pass time 12.33 +[Apr 26, 2026 - 01:47:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 2 : backward pass time 11.58 +[Apr 26, 2026 - 01:47:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 2 : optimizer step time 0.98 +[Apr 26, 2026 - 01:47:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 2-------------------- +[Apr 26, 2026 - 01:47:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[Apr 26, 2026 - 01:47:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[Apr 26, 2026 - 01:47:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 01:47:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 01:48:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.767495 38.812225 38.715862 +peak_gpu_mem_gb 62.354346 62.354346 62.354346 +peak_gpu_mem_reserved_gb 64.042969 64.042969 64.042969 +util 96.750000 99.000000 91.000000 +[Apr 26, 2026 - 01:48:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 3 : avg iteration time 51.59 seconds +[Apr 26, 2026 - 01:48:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 3 : data loading time 0.00 +[Apr 26, 2026 - 01:48:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 3 : avg forward pass time 13.73 +[Apr 26, 2026 - 01:48:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 3 : backward pass time 11.56 +[Apr 26, 2026 - 01:48:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 3 : optimizer step time 0.98 +[Apr 26, 2026 - 01:48:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 3-------------------- +[Apr 26, 2026 - 01:48:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0076 iter count: 1.0 +[Apr 26, 2026 - 01:48:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0076 iter count: 1.0 +[Apr 26, 2026 - 01:48:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 01:48:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 01:49:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.767725 38.812092 38.715878 +peak_gpu_mem_gb 62.354346 62.354346 62.354346 +peak_gpu_mem_reserved_gb 64.077148 64.121094 64.062500 +util 96.750000 99.000000 92.000000 +[Apr 26, 2026 - 01:49:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 4 : avg iteration time 46.18 seconds +[Apr 26, 2026 - 01:49:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 4 : data loading time 0.00 +[Apr 26, 2026 - 01:49:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 4 : avg forward pass time 11.05 +[Apr 26, 2026 - 01:49:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 4 : backward pass time 11.54 +[Apr 26, 2026 - 01:49:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 4 : optimizer step time 0.97 +[Apr 26, 2026 - 01:49:21 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 4-------------------- +[Apr 26, 2026 - 01:49:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[Apr 26, 2026 - 01:49:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[Apr 26, 2026 - 01:49:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 01:49:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 01:52:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.830145 38.873276 38.778778 +peak_gpu_mem_gb 126.559253 126.559253 126.559253 +peak_gpu_mem_reserved_gb 138.122070 138.166016 138.107422 +util 62.000000 67.000000 54.000000 +[Apr 26, 2026 - 01:52:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 5 : avg iteration time 176.62 seconds +[Apr 26, 2026 - 01:52:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 5 : data loading time 0.00 +[Apr 26, 2026 - 01:52:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 5 : avg forward pass time 79.12 +[Apr 26, 2026 - 01:52:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 5 : backward pass time 9.57 +[Apr 26, 2026 - 01:52:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 5 : optimizer step time 0.42 +[Apr 26, 2026 - 01:52:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 5-------------------- +[Apr 26, 2026 - 01:52:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 47557.1725 iter count: 1.0 +[Apr 26, 2026 - 01:52:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7875 iter count: 1.0 +[Apr 26, 2026 - 01:52:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 47557.1725iter count: 1.0 +[Apr 26, 2026 - 01:52:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 01:52:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 3.4437iter count: 1.0 +[Apr 26, 2026 - 01:52:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.8679 iter count: 1.0 +[Apr 26, 2026 - 01:52:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.0182 iter count: 1.0 +[Apr 26, 2026 - 01:52:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 60208.6602iter count: 1.0 +[Apr 26, 2026 - 01:52:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 5.6736 iter count: 1.0 +[Apr 26, 2026 - 01:52:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 1247158.0000iter count: 1.0 +[Apr 26, 2026 - 01:52:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7875iter count: 1.0 +[Apr 26, 2026 - 01:52:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.4145 iter count: 1.0 +[Apr 26, 2026 - 01:52:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 6829.5356iter count: 1.0 +[Apr 26, 2026 - 01:52:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.0395 iter count: 1.0 +[Apr 26, 2026 - 01:52:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 436.5892iter count: 1.0 +[Apr 26, 2026 - 01:52:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 6.4750 iter count: 1.0 +[Apr 26, 2026 - 01:52:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 420810.0938iter count: 1.0 +[Apr 26, 2026 - 01:52:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 1.9724 iter count: 1.0 +[Apr 26, 2026 - 01:52:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 51.6701 iter count: 1.0 +[Apr 26, 2026 - 01:52:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[Apr 26, 2026 - 01:52:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 01:52:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 26, 2026 - 01:53:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.830427 38.874321 38.778816 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.379395 67.437500 67.205078 +util 96.750000 100.000000 94.000000 +[Apr 26, 2026 - 01:53:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 6 : avg iteration time 51.88 seconds +[Apr 26, 2026 - 01:53:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 6 : data loading time 0.00 +[Apr 26, 2026 - 01:53:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 6 : avg forward pass time 13.92 +[Apr 26, 2026 - 01:53:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 6 : backward pass time 11.55 +[Apr 26, 2026 - 01:53:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 6 : optimizer step time 0.98 +[Apr 26, 2026 - 01:53:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 6-------------------- +[Apr 26, 2026 - 01:53:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0082 iter count: 1.0 +[Apr 26, 2026 - 01:53:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[Apr 26, 2026 - 01:53:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 01:53:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 01:54:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.830719 38.874554 38.778809 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.398926 67.400391 67.398438 +util 97.250000 99.000000 95.000000 +[Apr 26, 2026 - 01:54:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 7 : avg iteration time 51.42 seconds +[Apr 26, 2026 - 01:54:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 7 : data loading time 0.00 +[Apr 26, 2026 - 01:54:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 7 : avg forward pass time 13.67 +[Apr 26, 2026 - 01:54:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 7 : backward pass time 11.53 +[Apr 26, 2026 - 01:54:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 7 : optimizer step time 0.99 +[Apr 26, 2026 - 01:54:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 7-------------------- +[Apr 26, 2026 - 01:54:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0078 iter count: 1.0 +[Apr 26, 2026 - 01:54:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0078 iter count: 1.0 +[Apr 26, 2026 - 01:54:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 01:54:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 01:54:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.830716 38.874569 38.778816 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.330566 67.361328 67.320312 +util 97.000000 99.000000 95.000000 +[Apr 26, 2026 - 01:54:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 8 : avg iteration time 51.40 seconds +[Apr 26, 2026 - 01:54:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 8 : data loading time 0.00 +[Apr 26, 2026 - 01:54:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 8 : avg forward pass time 13.66 +[Apr 26, 2026 - 01:54:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 8 : backward pass time 11.55 +[Apr 26, 2026 - 01:54:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 8 : optimizer step time 0.98 +[Apr 26, 2026 - 01:54:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 8-------------------- +[Apr 26, 2026 - 01:54:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0074 iter count: 1.0 +[Apr 26, 2026 - 01:54:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0074 iter count: 1.0 +[Apr 26, 2026 - 01:54:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 01:54:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 26, 2026 - 01:55:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.830939 38.875225 38.778866 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.623535 68.357422 67.378906 +util 96.250000 100.000000 91.000000 +[Apr 26, 2026 - 01:55:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 9 : avg iteration time 46.01 seconds +[Apr 26, 2026 - 01:55:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 9 : data loading time 0.00 +[Apr 26, 2026 - 01:55:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 9 : avg forward pass time 11.00 +[Apr 26, 2026 - 01:55:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 9 : backward pass time 11.53 +[Apr 26, 2026 - 01:55:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 9 : optimizer step time 0.97 +[Apr 26, 2026 - 01:55:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 9-------------------- +[Apr 26, 2026 - 01:55:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0074 iter count: 1.0 +[Apr 26, 2026 - 01:55:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0074 iter count: 1.0 +[Apr 26, 2026 - 01:55:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 26, 2026 - 01:55:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260426_014355-3eo2d8ep/files/requirements.txt b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260426_014355-3eo2d8ep/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..a7117b8203d3b4df2b52c765d49a822a97b57cd9 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260426_014355-3eo2d8ep/files/requirements.txt @@ -0,0 +1,247 @@ +nvitop==1.6.1 +ftfy==6.3.1 +braceexpand==0.1.7 +antlr4-python3-runtime==4.9.3 +webdataset==1.0.2 +sentry-sdk==2.53.0 +rdkit==2025.9.5 +python-dotenv==1.2.1 +proglog==0.1.12 +omegaconf==2.3.0 +narwhals==2.17.0 +loguru==0.7.3 +imageio-ffmpeg==0.6.0 +plotly==6.5.2 +moviepy==2.2.1 +hydra-core==1.3.2 +toml==0.10.2 +wandb==0.25.0 +types-certifi==2021.10.8.3 +types-toml==0.10.8.20260408 +synchronicity==0.12.2 +propcache==0.4.1 +multidict==6.7.1 +hyperframe==6.1.0 +hpack==4.1.0 +frozenlist==1.8.0 +cbor2==5.9.0 +aiohappyeyeballs==2.6.1 +yarl==1.23.0 +watchfiles==1.1.1 +h2==4.3.0 +aiosignal==1.4.0 +grpclib==0.4.9 +aiohttp==3.13.5 +modal==1.4.1 +packaging==25.0 +setuptools==80.10.2 +wheel==0.46.3 +pip==26.0.1 +webencodings==0.5.1 +pure_eval==0.2.3 +ptyprocess==0.7.0 +nvidia-ml-py==13.590.48 +nvidia-cusparselt-cu12==0.7.1 +mpmath==1.3.0 +fastjsonschema==2.21.2 +zipp==3.23.0 +xyzservices==2025.11.0 +widgetsnbextension==4.0.15 +websocket-client==1.9.0 +webcolors==25.10.0 +wcwidth==0.6.0 +urllib3==2.6.3 +uri-template==1.3.0 +tzdata==2025.3 +typing_extensions==4.15.0 +triton==3.6.0 +traitlets==5.14.3 +tqdm==4.67.3 +tornado==6.5.5 +tinycss2==1.4.0 +sympy==1.14.0 +soupsieve==2.8.3 +smmap==5.0.3 +six==1.16.0 +sentencepiece==0.2.1 +Send2Trash==2.1.0 +safetensors==0.7.0 +rpds-py==0.30.0 +rfc3986-validator==0.1.1 +regex==2026.2.28 +pyzmq==27.1.0 +PyYAML==6.0.3 +python-json-logger==4.0.0 +Pygments==2.19.2 +pycparser==3.0 +psutil==7.2.2 +protobuf==4.24.4 +prometheus_client==0.24.1 +platformdirs==4.9.4 +pillow==11.3.0 +pexpect==4.9.0 +parso==0.8.6 +pandocfilters==1.5.1 +nvidia-nvtx-cu12==12.8.90 +nvidia-nvshmem-cu12==3.4.5 +nvidia-nvjitlink-cu12==12.8.93 +nvidia-nccl-cu12==2.27.5 +nvidia-curand-cu12==10.3.9.90 +nvidia-cufile-cu12==1.13.1.3 +nvidia-cuda-runtime-cu12==12.8.90 +nvidia-cuda-nvrtc-cu12==12.8.93 +nvidia-cuda-cupti-cu12==12.8.90 +nvidia-cublas-cu12==12.8.4.1 +numpy==1.26.4 +networkx==3.6.1 +nest-asyncio==1.6.0 +mistune==3.2.0 +MarkupSafe==3.0.3 +lark==1.3.1 +jupyterlab_widgets==3.0.16 +jupyterlab_pygments==0.3.0 +jsonpointer==3.0.0 +json5==0.13.0 +jmespath==1.1.0 +idna==3.11 +hf-xet==1.4.2 +h11==0.16.0 +fsspec==2026.2.0 +fqdn==1.5.1 +filelock==3.25.2 +executing==2.2.1 +einops==0.8.2 +defusedxml==0.7.1 +decorator==5.2.1 +debugpy==1.8.20 +cuda-pathfinder==1.4.2 +comm==0.2.3 +charset-normalizer==3.4.5 +certifi==2026.2.25 +bleach==6.3.0 +babel==2.18.0 +av==17.0.0 +attrs==25.4.0 +async-lru==2.2.0 +asttokens==3.0.1 +annotated-types==0.7.0 +typing-inspection==0.4.2 +terminado==0.18.1 +stack-data==0.6.3 +scipy==1.17.1 +rfc3987-syntax==1.1.0 +rfc3339-validator==0.1.4 +requests==2.32.5 +referencing==0.37.0 +python-dateutil==2.9.0.post0 +pydantic_core==2.41.5 +prompt_toolkit==3.0.52 +opencv-python-headless==4.11.0.86 +nvidia-cusparse-cu12==12.5.8.93 +nvidia-cufft-cu12==11.3.3.83 +nvidia-cudnn-cu12==9.10.2.21 +matplotlib-inline==0.2.1 +jupyter_core==5.9.1 +Jinja2==3.1.6 +jedi==0.19.2 +ipython_pygments_lexers==1.1.1 +importlib_metadata==8.7.1 +ImageIO==2.37.3 +httpcore==1.0.9 +gitdb==4.0.12 +cuda-bindings==12.9.4 +contourpy==1.3.3 +cffi==2.0.0 +beautifulsoup4==4.14.3 +anyio==4.12.1 +soundfile==0.13.1 +pydantic==2.12.5 +nvidia-cusolver-cu12==11.7.3.90 +jupyter_server_terminals==0.5.4 +jupyter_client==8.8.0 +jsonschema-specifications==2025.9.1 +ipython==9.11.0 +httpx==0.28.1 +GitPython==3.1.46 +botocore==1.42.68 +bokeh==3.9.0 +arrow==1.4.0 +argon2-cffi-bindings==25.1.0 +torch==2.10.0 +s3transfer==0.16.0 +jsonschema==4.26.0 +isoduration==20.11.0 +ipywidgets==8.1.8 +ipykernel==7.2.0 +argon2-cffi==25.1.0 +torchvision==0.25.0 +nbformat==5.10.4 +jupyter-console==6.6.3 +boto3==1.42.68 +accelerate==1.13.0 +nbclient==0.10.4 +jupyter-events==0.12.0 +nbconvert==7.17.0 +jupyter_server==2.17.0 +notebook_shim==0.2.4 +jupyterlab_server==2.28.0 +jupyter-lsp==2.3.0 +jupyterlab==4.5.6 +notebook==7.5.5 +jupyter==1.1.1 +fastgen==0.1.0 +pandas==3.0.1 +shellingham==1.5.4 +mdurl==0.1.2 +annotated-doc==0.0.4 +markdown-it-py==4.0.0 +rich==14.3.3 +typer==0.24.1 +huggingface_hub==1.7.1 +timm==1.0.25 +tokenizers==0.22.2 +diffusers==0.37.0 +transformers==5.3.0 +peft==0.18.1 +easydict==1.13 +lmdb==2.2.0 +threadpoolctl==3.6.0 +soxr==1.0.0 +msgpack==1.1.2 +llvmlite==0.47.0 +lazy-loader==0.5 +joblib==1.5.3 +audioread==3.1.0 +scikit-learn==1.8.0 +pooch==1.9.0 +numba==0.65.0 +librosa==0.11.0 +simsimd==6.5.16 +flatbuffers==25.12.19 +tifffile==2026.3.3 +stringzilla==4.6.0 +pyparsing==3.3.2 +prettytable==3.17.0 +onnx==1.17.0 +kiwisolver==1.5.0 +fonttools==4.62.1 +Cython==3.2.4 +cycler==0.12.1 +scikit-image==0.26.0 +onnxruntime==1.24.4 +matplotlib==3.10.8 +albucore==0.0.24 +albumentations==2.0.8 +insightface==0.7.3 +kornia_rs==0.1.10 +kornia==0.8.2 +pluggy==1.6.0 +iniconfig==2.3.0 +pytest==8.1.1 +dlib==20.0.1 +lpips==0.1.4 +python_speech_features==0.6 +decord==0.6.0 +click==8.2.1 +scenedetect==0.6.7.1 +torchaudio==2.10.0 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260426_014355-3eo2d8ep/files/wandb-metadata.json b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260426_014355-3eo2d8ep/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..2880c55427a60f55bd0db2651e501f118a5935e9 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260426_014355-3eo2d8ep/files/wandb-metadata.json @@ -0,0 +1,72 @@ +{ + "os": "Linux-5.15.0-151-generic-x86_64-with-glibc2.39", + "python": "CPython 3.12.12", + "startedAt": "2026-04-25T16:43:55.700882Z", + "args": [ + "--config=fastgen/configs/experiments/OmniAvatar/config_sf_sink1_window7_redmd_beta2_taew.py", + "-", + "trainer.resume=False", + "model.fake_score_optimizer.lr=3e-6", + "log_config.group=omniavatar_sf_audiofix", + "log_config.name=sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769", + "log_config.project=OmniAvatar-FastGen", + "log_config.wandb_entity=paulhcho", + "model.sample_t_cfg.t_list=[0.999,0.769,0.0]" + ], + "program": "/home/work/.local/hyunbin/FastGen-redmd/train.py", + "codePath": "train.py", + "codePathLocal": "train.py", + "git": { + "remote": "https://paulcho98:@github.com/paulcho98/FastGen.git", + "commit": "302cf2d9ddcceacc8fca5e480cb21b26ea6c07be" + }, + "email": "paul.hyunbin@gmail.com", + "root": "/tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769", + "host": "main1", + "executable": "/home/work/.local/miniconda3/envs/hb_fastgen/bin/python3.12", + "cpu_count": 112, + "cpu_count_logical": 224, + "gpu": "NVIDIA H200", + "gpu_count": 4, + "disk": { + "/": { + "total": "1356758433792", + "used": "979257122816" + } + }, + "memory": { + "total": "2163961778176" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-4685d4b3-5cf9-2766-43d3-b9615a684b7c" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ec888a66-4b6f-b8de-b34b-249efb9ad262" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-9c1e1773-d710-06c9-7db7-1b54e9fc3790" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2b1017dc-2958-a946-16d2-2c29da6d18b0" + } + ], + "cudaVersion": "12.9", + "writerId": "upk6yt8d6gxnny9vydatyl2lnqzs7oye" +} \ No newline at end of file diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260426_014355-3eo2d8ep/files/wandb-summary.json b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260426_014355-3eo2d8ep/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..8548a5a5d746499efb3a57cea92c165426e73a5e --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260426_014355-3eo2d8ep/files/wandb-summary.json @@ -0,0 +1 @@ +{"train/reward_sync_c_r3":1.9724397659301758,"net/trainable_params":1421383840,"net/local_trainable_params":[376675360,376398880,376398880,376398880],"fake_score/trainable_params":174981120,"_timestamp":1.7771361384025977e+09,"train/reward_weight_r2":420810.09375,"profiler/backward_pass_time":11.52579953148961,"train/reward_weight_r1":436.5892333984375,"_runtime":735.054590651,"fake_score/total_params":1596364960,"optimizer/lr_net":2e-06,"optimizer/iteration":9,"fake_score/local_total_params":[399300904,399021352,399021352,399021352],"fake_score/local_trainable_params":[43745280,43745280,43745280,43745280],"train/reward_sync_c_r0":4.414505958557129,"net/total_params":1421383840,"train/vsd_loss_unweighted":0.7874755859375,"train/vsd_loss":0.7874755859375,"train/reward_weight_r3":51.67011260986328,"train/reward_weight_max":1.247158e+06,"train/reward_weight_mean":60208.66015625,"train/total_loss":0.00740814208984375,"profiler/avg_forward_pass_time":11.003798618912697,"train/reward_sync_c_r2":6.474968433380127,"train/vsd_loss_weighted":47557.17248535156,"train/fake_score_loss":0.00740814208984375,"optimizer/grad_norm (model_key net)":0,"_wandb":{"runtime":735},"train/gan_loss_disc":0,"model/trainable_params":1596364960,"net/local_total_params":[376675360,376398880,376398880,376398880],"optimizer/lr_fake_score":3e-06,"profiler/optimizer_step_time":0.9744416121393442,"model/local_trainable_params":[420420640,420144160,420144160,420144160],"train/reward_weight_r0":6829.53564453125,"model/local_total_params":[4364057584,4362262512,4362262512,4362262512],"train/reward_sync_c_max":7.018188953399658,"profiler/data_loading_time":0.0008797124028205872,"_step":9,"train/reward_sync_c_r1":3.039496421813965,"train/reward_weight_min":5.673583030700684,"optimizer/non_finite_grads_count (model_key net)":0,"train/reward_sync_c_mean":3.443650245666504,"profiler/avg_iteration_time":46.01223087310791,"train/reward_sync_c_min":0.8679103851318359,"train/gan_loss_gen":0,"model/total_params":17325462112} \ No newline at end of file diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260426_014355-3eo2d8ep/logs/debug-core.log b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260426_014355-3eo2d8ep/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..930b4d33fc8e99de52900cde3e1aa06eae71e622 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260426_014355-3eo2d8ep/logs/debug-core.log @@ -0,0 +1,13 @@ +{"time":"2026-04-26T01:43:55.802913389+09:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpdyg4ql84/port-1719007.txt","pid":1719007,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-04-26T01:43:55.803376615+09:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":1719007} +{"time":"2026-04-26T01:43:55.803365704+09:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1719007-1727139-1216689072/socket","Net":"unix"}} +{"time":"2026-04-26T01:43:55.98588454+09:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-04-26T01:43:56.001991952+09:00","level":"INFO","msg":"handleInformInit: received","streamId":"3eo2d8ep","id":"1(@)"} +{"time":"2026-04-26T01:43:56.391143829+09:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"3eo2d8ep","id":"1(@)"} +{"time":"2026-04-26T01:44:02.576731108+09:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"7ovmpx0k0wcn"} +{"time":"2026-04-26T01:56:12.374996965+09:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-04-26T01:56:12.375046189+09:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-04-26T01:56:12.375035416+09:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-04-26T01:56:12.37508681+09:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-04-26T01:56:12.375108934+09:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-1719007-1727139-1216689072/socket","Net":"unix"}} +{"time":"2026-04-26T01:56:12.587028978+09:00","level":"INFO","msg":"server: parent process exited, terminating service process"} diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260426_014355-3eo2d8ep/logs/debug-internal.log b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260426_014355-3eo2d8ep/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..abbaea6a19c902916474449ac42f7878aebd8076 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260426_014355-3eo2d8ep/logs/debug-internal.log @@ -0,0 +1,7 @@ +{"time":"2026-04-26T01:43:56.002079087+09:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-04-26T01:43:56.391025815+09:00","level":"INFO","msg":"stream: created new stream","id":"3eo2d8ep"} +{"time":"2026-04-26T01:43:56.391073608+09:00","level":"INFO","msg":"handler: started","stream_id":"3eo2d8ep"} +{"time":"2026-04-26T01:43:56.391137418+09:00","level":"INFO","msg":"stream: started","id":"3eo2d8ep"} +{"time":"2026-04-26T01:43:56.391155918+09:00","level":"INFO","msg":"writer: started","stream_id":"3eo2d8ep"} +{"time":"2026-04-26T01:43:56.391155304+09:00","level":"INFO","msg":"sender: started","stream_id":"3eo2d8ep"} +{"time":"2026-04-26T01:56:12.375039412+09:00","level":"INFO","msg":"stream: closing","id":"3eo2d8ep"} diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260426_014355-3eo2d8ep/logs/debug.log b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260426_014355-3eo2d8ep/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..a4ed3e08dffbfd94107435102fc2f1e79e1b0e7d --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260426_014355-3eo2d8ep/logs/debug.log @@ -0,0 +1,21 @@ +2026-04-26 01:43:55,702 INFO MainThread:1719007 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-04-26 01:43:55,702 INFO MainThread:1719007 [wandb_setup.py:_flush():81] Configure stats pid to 1719007 +2026-04-26 01:43:55,702 INFO MainThread:1719007 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-04-26 01:43:55,702 INFO MainThread:1719007 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260426_014355-3eo2d8ep/logs/debug.log +2026-04-26 01:43:55,702 INFO MainThread:1719007 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260426_014355-3eo2d8ep/logs/debug-internal.log +2026-04-26 01:43:55,702 INFO MainThread:1719007 [wandb_init.py:init():844] calling init triggers +2026-04-26 01:43:55,702 INFO MainThread:1719007 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'log_config': {'project': 'OmniAvatar-FastGen', 'group': 'omniavatar_sf_audiofix', 'name': 'sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769', 'wandb_mode': 'online', 'wandb_entity': 'paulhcho', 'wandb_credential': './credentials/wandb_api.txt'}, 'trainer': {'cudnn': {'deterministic': 'False', 'benchmark': 'True'}, 'checkpointer': {'save_dir': '/tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints', 'use_s3': 'False', 's3_container': 's3://checkpoints/fastgen', 's3_credential': './credentials/s3.json', 'pretrained_ckpt_path': '/home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth', 'pretrained_ckpt_key_map': {'net': 'net'}}, 'callbacks': {'grad_clip': {'grad_norm': '10.0', 'model_key': 'net', '_target_': ""}, 'gpu_stats': {'every_n': '100', '_target_': ""}, 'train_profiler': {'every_n': '100', '_target_': ""}, 'param_count': {'_target_': ""}, 'ema': {'type': 'constant', 'beta': '0.9999', 'gamma': '16.97', 'ema_halflife_kimg': '500', 'ema_rampup_ratio': '0.05', 'start_iter': '0', '_target_': ""}, 'wandb': {'sample_logging_iter': '100', '_target_': "", 'fps': '25', 'syncnet_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model'}}, 'save_ckpt_iter': '100', 'validation_iter': '100', 'skip_initial_validation': 'True', 'logging_iter': '1', 'max_iter': '5000', 'visualize_teacher': 'False', 'seed': '0', 'val_seed': None, 'resume': 'False', 'ddp': 'False', 'fsdp': 'True', 'tf32_enabled': 'True', 'grad_accum_rounds': '2', 'batch_size_global': None, 'offload_module_in_decoding': 'False', 'fsdp_cpu_offload': 'False', 'fsdp_min_num_params': '10000000', 'fsdp_sharding_group_size': None, 'global_vars': None, 'global_vars_val': [None], 'augment_pipe': None}, 'dataloader_train': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_path.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '8', 'num_workers': '2', 'neg_text_emb_path': None, 'use_ref_sequence': 'True', '_target_': "", 'load_raw_audio': 'True', 'raw_audio_sample_rate': '16000', 'raw_audio_num_frames': '81', 'raw_audio_fps': '25.0'}, 'dataloader_val': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_val10.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '1', 'num_workers': '2', 'neg_text_emb_path': None, 'use_ref_sequence': 'True', 'load_ode_path': 'False', '_target_': ''}, 'eval': {'num_samples': '50000', 'save_images': 'False', 'min_ckpt': '0', 'max_ckpt': '100000000', 'samples_dir': 'samples'}, 'model': {'net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'chunk_size': '3', 'total_num_frames': '21', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt', 'net_pred_type': 'flow', 'schedule_type': 'rf', 'local_attn_size': '7', 'sink_size': '1', 'use_dynamic_rope': 'True', '_target_': ""}, 'teacher': {'model_size': '14B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'fake_score_net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt', 'merge_lora': 'False', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'guidance_scale': '4.5', 'timestep_cfg': {'enabled': 'True', 't_lo': '0.556', 't_hi': '0.882'}, 'skip_layers': None, 'net_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'net_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'sample_t_cfg': {'time_dist_type': 'shifted', 'train_p_mean': '-1.1', 'train_p_std': '2.0', 'shift': '5.0', 'min_t': '0.001', 'max_t': '0.999', 't_list': ['0.999', '0.769', '0.0'], 'log_t_df': '0.01'}, 'input_shape': ['16', '21', '64', '64'], 'device': 'cuda', 'grad_scaler_enabled': 'False', 'grad_scaler_init_scale': '65536.0', 'grad_scaler_growth_interval': '2000', 'pretrained_model_path': '', 'pretrained_student_net_path': '', 'load_student_weights': 'False', 'enable_preprocessors': 'True', 'use_ema': 'False', 'student_sample_steps': '2', 'student_sample_type': 'sde', 'fsdp_meta_init': 'False', 'add_teacher_to_fsdp_dict': 'True', 'ddp_find_unused_parameters': 'True', 'precision': 'bfloat16', 'precision_amp': None, 'precision_amp_infer': None, 'precision_amp_enc': None, 'precision_fsdp': 'float32', 'fake_score_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '3e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'fake_score_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'discriminator': {'feature_indices': '{0, 1, 2}', 'all_res': ['32', '16', '8'], 'in_channels': '256', '_target_': ""}, 'discriminator_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '0.0001', 'weight_decay': '0.01', 'betas': ['0.9', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'discriminator_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'student_update_freq': '5', 'gan_loss_weight_gen': '0', 'gan_use_same_t_noise': 'False', 'fake_score_pred_type': 'x0', 'gan_r1_reg_weight': '0.0', 'gan_r1_reg_alpha': '0.1', 'enable_gradient_in_rollout': 'True', 'start_gradient_frame': '0', 'same_step_across_blocks': 'True', 'last_step_only': 'False', 'context_noise': '0.0', 'fake_score': None, 'reward': {'enabled': 'True', 'checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model', 'input_fps': '25.0', 'audio_sample_rate': '16000', 'vshift': '15', 'decoder_kind': 'taew', 'taew_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth'}, 'reward_beta': '2', 'center_reward': 'False', 'clamp_reward': None, 'reward_weighting_mode': 'per_sample', 'save_reward_debug_video': 'False', 'reward_debug_dir': 'logs/redmd_debug'}, 'model_class': {'config': None, '_target_': ""}, '_wandb': {}} +2026-04-26 01:43:55,702 INFO MainThread:1719007 [wandb_init.py:init():892] starting backend +2026-04-26 01:43:55,986 INFO MainThread:1719007 [wandb_init.py:init():895] sending inform_init request +2026-04-26 01:43:56,000 INFO MainThread:1719007 [wandb_init.py:init():903] backend started and connected +2026-04-26 01:43:56,004 INFO MainThread:1719007 [wandb_init.py:init():973] updated telemetry +2026-04-26 01:43:56,024 INFO MainThread:1719007 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-04-26 01:43:57,318 INFO MainThread:1719007 [wandb_init.py:init():1042] starting run threads in backend +2026-04-26 01:43:57,568 INFO MainThread:1719007 [wandb_run.py:_console_start():2524] atexit reg +2026-04-26 01:43:57,568 INFO MainThread:1719007 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-04-26 01:43:57,568 INFO MainThread:1719007 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-04-26 01:43:57,568 INFO MainThread:1719007 [wandb_run.py:_redirect():2465] Redirects installed. +2026-04-26 01:43:57,574 INFO MainThread:1719007 [wandb_init.py:init():1082] run started, returning control to user process +2026-04-26 01:56:12,375 INFO wandb-AsyncioManager-main:1719007 [service_client.py:_forward_responses():134] Reached EOF. +2026-04-26 01:56:12,375 INFO wandb-AsyncioManager-main:1719007 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles. diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260426_014355-3eo2d8ep/run-3eo2d8ep.wandb b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260426_014355-3eo2d8ep/run-3eo2d8ep.wandb new file mode 100644 index 0000000000000000000000000000000000000000..7dd7cfab927919b6e9c62f0160e0a0563c0bbd3d --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260426_014355-3eo2d8ep/run-3eo2d8ep.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e1f6ae9c95bfa12616133754c80ac14cca4c7496e58a0cec15f770041863dff +size 163840 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/train_media/data/real_100_47403c800591fce4a99b.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/train_media/data/real_100_47403c800591fce4a99b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23f2f377ccf704bcaa09a4c9b1a9b5151ab73aac --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/train_media/data/real_100_47403c800591fce4a99b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47403c800591fce4a99b38622b7607baeec11f4e025df3962d292acf880fba6c +size 192248 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/train_media/data/real_200_4e05a85e239f00a00e38.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/train_media/data/real_200_4e05a85e239f00a00e38.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c0fdfbad50d1423fb0ee98a75767d17d3c13b75b --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/train_media/data/real_200_4e05a85e239f00a00e38.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e05a85e239f00a00e386c8d8a9eaabeac26dd4943226f41d80d9f6b7c50bff6 +size 181830 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/train_media/data/real_300_97779d11e00d4fea84a0.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/train_media/data/real_300_97779d11e00d4fea84a0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1fd7a958d162e551e0f6ce9b94e075ce8f98807e --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/train_media/data/real_300_97779d11e00d4fea84a0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97779d11e00d4fea84a00b1667ca4d081ba87b85cf65fac40d228e9774f762d2 +size 194172 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/train_media/data/real_400_112c0905448a821974f0.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/train_media/data/real_400_112c0905448a821974f0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9684c7bde9ed18086d39e7a9880b642c505b3947 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/train_media/data/real_400_112c0905448a821974f0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:112c0905448a821974f0bf0a94d9107473bb02b60e0ebe1e75a57d309670facb +size 219311 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/train_media/data/real_500_2594e32c120a620ede9d.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/train_media/data/real_500_2594e32c120a620ede9d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7e38036c10e966c2c82fa717b29431a1ff88e1ae --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/train_media/data/real_500_2594e32c120a620ede9d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2594e32c120a620ede9d4613c87c8985d5ec03a9250077d94093cf8001922aae +size 190461 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/train_media/data/real_600_aebf0f3e2314a893fcf4.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/train_media/data/real_600_aebf0f3e2314a893fcf4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..df584863079059c25acb730ed404066fdef5d340 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/train_media/data/real_600_aebf0f3e2314a893fcf4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aebf0f3e2314a893fcf47216a0d6c68b02b30d80a7347c759681d9497a805c01 +size 220015 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/train_media/student/generation_100_9ad898f857d040bf9b70.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/train_media/student/generation_100_9ad898f857d040bf9b70.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..79d1848cd13cd0f7a5d9ea9365397d362c5e02a2 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/train_media/student/generation_100_9ad898f857d040bf9b70.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ad898f857d040bf9b705b126cb9e3c40b05b749e922c5232c4d01359ca8dd6d +size 221798 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/train_media/student/generation_200_ec698bf06b6b445b4e9e.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/train_media/student/generation_200_ec698bf06b6b445b4e9e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c336619d073c078965d69c7bb3ab8a9ccc373738 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/train_media/student/generation_200_ec698bf06b6b445b4e9e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec698bf06b6b445b4e9eff5945405ae7793ec8e9266db4d0fe1efc97fbc51901 +size 212885 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/train_media/student/generation_300_529218f9ad0b4522c2ed.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/train_media/student/generation_300_529218f9ad0b4522c2ed.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..32de0acbf251e268dcd2dc72bc5a17d40cfee44c --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/train_media/student/generation_300_529218f9ad0b4522c2ed.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:529218f9ad0b4522c2ed5d0927b88567d888716150092254075a4ee562e7ceda +size 249130 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/train_media/student/generation_400_0da984eac02beebf8e3d.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/train_media/student/generation_400_0da984eac02beebf8e3d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..89b69d50d725acbbe415efafeed0149a4c6cdd6e --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/train_media/student/generation_400_0da984eac02beebf8e3d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0da984eac02beebf8e3d7db352ada5a72764d38fb655d480c51de398c1737b77 +size 321379 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/train_media/student/generation_500_d122305464d3ebb64647.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/train_media/student/generation_500_d122305464d3ebb64647.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1eb51fc2f90fb587610f52981d3e438000e26e59 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/train_media/student/generation_500_d122305464d3ebb64647.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d122305464d3ebb6464798dbc125692199054e210e67ecbc5dd940870b52c1ea +size 288645 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/train_media/student/generation_600_16faed3c3d83d799f457.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/train_media/student/generation_600_16faed3c3d83d799f457.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d7ecd67877ba581690d42e0d4f6cd4f71e0c29cc --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/train_media/student/generation_600_16faed3c3d83d799f457.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16faed3c3d83d799f457213f63d54c372d06e46f930eb3b293194c4901bb11c3 +size 276496 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_100_0339428f7329a7be4e09.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_100_0339428f7329a7be4e09.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..34fe50aeab57b50236e449f995a9532a05b5838d --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_100_0339428f7329a7be4e09.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0339428f7329a7be4e095ed08fc692a08ab9bd915f32f0d5fd73e3908ee3a2cb +size 184843 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_100_1df6e87dc4f30c2902b7.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_100_1df6e87dc4f30c2902b7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..373ae0a39dc5634f831735caea121468e152c2a5 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_100_1df6e87dc4f30c2902b7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1df6e87dc4f30c2902b77217538abadb61e567561db3f38e2e2362ab0b3d23c4 +size 295982 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_100_79c7e7fe822f87014a1f.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_100_79c7e7fe822f87014a1f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..671b97d0dfa3afc17ec9c3568d0ab14e7f058ddc --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_100_79c7e7fe822f87014a1f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79c7e7fe822f87014a1f6c49c7b9bbb7365ae4b32ffb6f9725877aaf43a5d17f +size 246984 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_100_8143de6481ab40f8b120.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_100_8143de6481ab40f8b120.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fcd6c0338bd208c714edaacfa77a43c92e1a0eb7 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_100_8143de6481ab40f8b120.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8143de6481ab40f8b120a3c07e434d21a840265b6d6c5ad63656d66f3bf0c826 +size 308418 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_100_86cb2c4f20d248423637.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_100_86cb2c4f20d248423637.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b8e926acb10cf80b3dfff002aa0ffe6b1a978ee3 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_100_86cb2c4f20d248423637.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86cb2c4f20d2484236371aaed7bbb1e1ed67dc22b30089150ae54c811fffc5c5 +size 257733 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_100_9415105b25edacb0459a.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_100_9415105b25edacb0459a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f0db7ebe6c59d1b08a0690d90d02f22a639e9b48 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_100_9415105b25edacb0459a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9415105b25edacb0459a148ae730595d1455bf97ed16ebd9525dffe553e5c513 +size 214335 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_100_bbdbcef0aac20e14f029.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_100_bbdbcef0aac20e14f029.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0aba970b6e7f1d357315597d77aed3e00ac98635 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_100_bbdbcef0aac20e14f029.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bbdbcef0aac20e14f029333ade4e267af6729fb8fae6e334830d25ac26b8286f +size 245240 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_100_be9c98f50bcc5960eb6b.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_100_be9c98f50bcc5960eb6b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9f75674d0769d1be8f2c857cc54b684ceea51ffe --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_100_be9c98f50bcc5960eb6b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be9c98f50bcc5960eb6b77cfb62868cab95660d42159533484dd98af8c2f5fc0 +size 222198 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_100_eebd5c16e611688d18b2.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_100_eebd5c16e611688d18b2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e415219e62d9393cfade26567c271e45e960121c --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_100_eebd5c16e611688d18b2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eebd5c16e611688d18b2a85049955a96ee058393e4ce41b924a293b209f20f8c +size 201616 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_100_f77102c7b3d9fe62ef24.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_100_f77102c7b3d9fe62ef24.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..139c800c97fd2f7d0f46199c136fc5c6e490c8bd --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_100_f77102c7b3d9fe62ef24.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f77102c7b3d9fe62ef2405b681de04aa44fc6ed8cc8a4050ca5515683ee0866e +size 202510 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_200_04334af08f58cf39aef4.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_200_04334af08f58cf39aef4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..acec544eb549af346fa271a3a5f717f186e8a398 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_200_04334af08f58cf39aef4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:04334af08f58cf39aef441f30c4e3b0e45bd7de13df3ed1bd2e5e015e2947ed5 +size 205428 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_200_3113f2a565b230be7f0e.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_200_3113f2a565b230be7f0e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7a456663d1c468b27ff16a662de75184ed4d2010 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_200_3113f2a565b230be7f0e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3113f2a565b230be7f0e6ee42d454d3ff8f25c62c9d4614eeb16e5d4b1336b40 +size 216647 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_200_4fa61c7c0dcdfbde8849.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_200_4fa61c7c0dcdfbde8849.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ae01038ff55ec716ec438c511db2a9c9b5175a53 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_200_4fa61c7c0dcdfbde8849.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4fa61c7c0dcdfbde8849d5a8bdaf545a7803ca29765f420ad56251b0c2409de6 +size 198794 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_200_510833ef7d8e361e104d.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_200_510833ef7d8e361e104d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..00c118cec8567d49b46b67b730c61bc31e775022 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_200_510833ef7d8e361e104d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:510833ef7d8e361e104d91dd389c351da1da41485225e2439fb153684fcb9c6a +size 242318 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_200_516378fb4f514f2526a6.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_200_516378fb4f514f2526a6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c126d84c7894115fec4d9d7d996aba773e80a1f1 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_200_516378fb4f514f2526a6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:516378fb4f514f2526a6256e8d2c78b03acc616c39040e7d2e579527b1bb2a70 +size 244566 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_200_a2d95f36c14a032f6dbb.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_200_a2d95f36c14a032f6dbb.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..48d382c7e2ec68518140a1a62352d07b0b5e30d4 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_200_a2d95f36c14a032f6dbb.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a2d95f36c14a032f6dbb3bcb1aa37266efcd011d89c8919533f194d9dfba889f +size 298253 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_200_a91df4fe1abceb619c6a.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_200_a91df4fe1abceb619c6a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..30d7f23b66b654cada3f6e003e6be25963643eab --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_200_a91df4fe1abceb619c6a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a91df4fe1abceb619c6a9a1e50a14dc955ad8f6b3dc5bb9a4dcc1fe9d7c98755 +size 287393 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_200_ce354f7516e45328e3eb.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_200_ce354f7516e45328e3eb.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c36580e06ba8ddc048147e34fe94fbef7291e947 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_200_ce354f7516e45328e3eb.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce354f7516e45328e3eb8c4570b6bc8ec0a522ad9adee6217a72cbd2fab617c5 +size 189929 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_200_da1eeeb78af6f345da3d.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_200_da1eeeb78af6f345da3d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..da452574f44d2df3f1e8e7bfb9416b622e9f914a --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_200_da1eeeb78af6f345da3d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da1eeeb78af6f345da3dab8adc4ff44a1cef5cc7d6e2cd55a951cb746a1bf149 +size 253026 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_200_f5a4d6c3a812176408fa.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_200_f5a4d6c3a812176408fa.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ecafbd796025b02d9307b47a247d51ef6a9355c3 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_200_f5a4d6c3a812176408fa.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5a4d6c3a812176408fa9bd147a2e113b580f60ef6aed1648dca10df1ae27390 +size 216431 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_300_1565131a5fdf095f0f38.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_300_1565131a5fdf095f0f38.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..479ee571f8a6189c956203060975b6e58685026b --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_300_1565131a5fdf095f0f38.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1565131a5fdf095f0f38ca95adc4c325709ec51ae85b81054578a73c453e8b89 +size 240623 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_300_30c8a4cb027f0417640c.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_300_30c8a4cb027f0417640c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..48ed4f7e70e6f3a45d72aeeadc894d2af2667e8c --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_300_30c8a4cb027f0417640c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30c8a4cb027f0417640ce5d757019ff711947b305ed9d19218a52b13c72f44c7 +size 210117 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_300_3782221edf7d8217e421.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_300_3782221edf7d8217e421.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..20c670729e1b995c46fb75a8b39bbaf109cacadb --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_300_3782221edf7d8217e421.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3782221edf7d8217e421590c5cf6921a9f634c4df66aa7047e772dfb06a5e4a8 +size 265084 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_300_4584f1b171cbf943b470.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_300_4584f1b171cbf943b470.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4d0110d1553b3134d2475129b002887326b7a2c3 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_300_4584f1b171cbf943b470.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4584f1b171cbf943b470b289ad3a5b517a75654f604d27588229a4a20a30429b +size 222973 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_300_4d698752082729725cd8.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_300_4d698752082729725cd8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6e071565757e7286af16bfd46d9702a73635452f --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_300_4d698752082729725cd8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d698752082729725cd859e166b88730b073a5484c58fef972d4b88878732000 +size 283132 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_300_60d9057ec37171fc7d5e.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_300_60d9057ec37171fc7d5e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..552b7fb1288fee8bfefa4f06ce0ab65a70900409 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_300_60d9057ec37171fc7d5e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60d9057ec37171fc7d5eff67e58acfca3fd5137e11675fb742f7f09fd1678d95 +size 235927 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_300_81c65606c72d6c7d8cc6.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_300_81c65606c72d6c7d8cc6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..baa3a294e7fb7a1dd2ae38dd62b415b843f6f9ba --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_300_81c65606c72d6c7d8cc6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81c65606c72d6c7d8cc60c53c5945a67538dee568b8b0b079cbea5f1ea816cbb +size 332738 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_300_820ff8c5f6bb7ed3f6fb.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_300_820ff8c5f6bb7ed3f6fb.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..50bf67474aaa255478c54facf423ea4f4b73e809 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_300_820ff8c5f6bb7ed3f6fb.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:820ff8c5f6bb7ed3f6fb292e448c4ebad3097f57076640c6147ae1a9fddfc097 +size 320741 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_300_874abc5f12e40930cdfd.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_300_874abc5f12e40930cdfd.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..acf43d569ca5998694d00ab83052329b870f0c51 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_300_874abc5f12e40930cdfd.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:874abc5f12e40930cdfd3d4024c491380522690c7587ee81728bba88f6bdcf87 +size 273455 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_300_e5472b72768a8bca34a0.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_300_e5472b72768a8bca34a0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f0bdb7643d7acb12e0bfcdafb055dd51acde9d96 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_300_e5472b72768a8bca34a0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5472b72768a8bca34a05f074d558a2e72fe8d10fdab25d14ba79898c7a09aeb +size 247278 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_400_0c07016fdc70f327cc1e.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_400_0c07016fdc70f327cc1e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cd7b67a364b19ea9be6f676ccef42816b2140777 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_400_0c07016fdc70f327cc1e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c07016fdc70f327cc1e7ffa5c5963ef2b365841349608d1b3ca4755ba7bc484 +size 372592 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_400_16406ece75a2e125e05b.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_400_16406ece75a2e125e05b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..97025547c3a5eae904a7052b47014f7db1d80eb8 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_400_16406ece75a2e125e05b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16406ece75a2e125e05be5c4e8eb7c3ac8f56e6c4be86bd45f1334fc6bc1d5cc +size 274889 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_400_36b0e00e14ad762f880f.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_400_36b0e00e14ad762f880f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7b205807969891cf2959da51dfa4d94fe8351023 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_400_36b0e00e14ad762f880f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36b0e00e14ad762f880ff2ffe802a028b8ee37e00b0f3d6db23a1c9ab8d6a477 +size 257414 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_400_4411bcf9d81d5de2a351.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_400_4411bcf9d81d5de2a351.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..05488b734e61133fd21a8a2945ba168cccf3f65d --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_400_4411bcf9d81d5de2a351.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4411bcf9d81d5de2a351b470976f4eeed7e904488faa9c3e54b5c84a986a093d +size 314543 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_400_67e3449dc772bbbb398d.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_400_67e3449dc772bbbb398d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b07564808fcd9fd332494f1e988e140c2a3779b4 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_400_67e3449dc772bbbb398d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67e3449dc772bbbb398d4979e3a4530100ad7ad07ecb2d785f8225364d502b57 +size 249195 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_400_7743f87671b0c2293457.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_400_7743f87671b0c2293457.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e661ee99a687921c4dae34880ad3b5cbf90abe8a --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_400_7743f87671b0c2293457.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7743f87671b0c22934573072ee9495d9ba4739f29210de179fba210f56f99454 +size 329942 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_400_869fb4c31411e732bd65.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_400_869fb4c31411e732bd65.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5e252d9e79e05dc32d614f1fb36208a39d6aa09c --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_400_869fb4c31411e732bd65.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:869fb4c31411e732bd65ce283cca3ec13b802639b93961fa3b2d66d7a2899cae +size 288717 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_400_9acc695f67b9d5a96303.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_400_9acc695f67b9d5a96303.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e3177935b7d73889c642de3ab0a4c96d1546992d --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_400_9acc695f67b9d5a96303.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9acc695f67b9d5a96303711ace96bbe6c79a055b0d750d72f0014515017eb7ae +size 374357 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_400_c634c350c0c74c3af278.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_400_c634c350c0c74c3af278.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7a2ec223895c0c46d7d2399965e8c50d1335781a --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_400_c634c350c0c74c3af278.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c634c350c0c74c3af27808a7647e90602401dd02d89b466613368218ff2d25c2 +size 274896 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_400_eeadc0c6b1921f934b7b.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_400_eeadc0c6b1921f934b7b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c862aa27e392d20e39500da1bb5c530f39353499 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_400_eeadc0c6b1921f934b7b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eeadc0c6b1921f934b7baf17eb3a8140fb56ae766f48c9d59a94e00ee220aafb +size 303348 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_500_41f749a7b0dfc091bd0b.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_500_41f749a7b0dfc091bd0b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..16e439ed22e542fc5e4197c9d34aa8b0ae6426cb --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_500_41f749a7b0dfc091bd0b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:41f749a7b0dfc091bd0b464f5e9802d4a035935de43cbec59986b5dcb45f9c5f +size 253146 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_500_4e9bfad62f5eeb28a51d.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_500_4e9bfad62f5eeb28a51d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..412c56f14b04e3ce445b0faa1eb998cb7343672c --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_500_4e9bfad62f5eeb28a51d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e9bfad62f5eeb28a51d3cc9e316766d82e4e8862a740bed61ef5ec2c2a9b698 +size 277850 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_500_556c9d12337a9883ffaa.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_500_556c9d12337a9883ffaa.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..484ed51c82b09b1b6fc98eb866885c1fc8c88053 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_500_556c9d12337a9883ffaa.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:556c9d12337a9883ffaacae59bf7757c03d9bbc79512202da0be746927dcd6e2 +size 277707 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_500_628aa97b42a1200127a0.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_500_628aa97b42a1200127a0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6a6031882b41f04678848477f40bf788b93a1608 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_500_628aa97b42a1200127a0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:628aa97b42a1200127a0a7b48cfa3359388a47c1a0103b8da139db9aecbc8286 +size 306237 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_500_70bb27d15c124e9f0f2b.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_500_70bb27d15c124e9f0f2b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1f1d536eaa8b05650d8f5274a06f787a862d7b80 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_500_70bb27d15c124e9f0f2b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:70bb27d15c124e9f0f2be56493e98b25719658ee6faa2556c1fd79986b7ccb11 +size 261772 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_500_813f28f2e9c8ee3a8277.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_500_813f28f2e9c8ee3a8277.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..53017f0b38111d914103b2bef8fd2d9e1e6640a1 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_500_813f28f2e9c8ee3a8277.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:813f28f2e9c8ee3a8277d9077121c0a725513b739fe4ac9adfb1cf0f65da2a27 +size 317093 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_500_8b2afa111a3cb9c38644.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_500_8b2afa111a3cb9c38644.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1e9b1de9b81c1c84d6e7452ce563b9cb872f2e2e --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_500_8b2afa111a3cb9c38644.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b2afa111a3cb9c386440a3e2f13014fdd2de1842efe3af3596b483159fecbb1 +size 376970 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_500_d893a7c5a5d3bf77e987.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_500_d893a7c5a5d3bf77e987.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b1655dd3e96e8e5a30f2a11f2ba68e0bea3d8bda --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_500_d893a7c5a5d3bf77e987.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d893a7c5a5d3bf77e9871c7d2271b53f871449c0bf6227449f964016e1f13739 +size 384017 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_500_d9aa9f2aa1576ba26432.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_500_d9aa9f2aa1576ba26432.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9c9cb5df79a89bd18db2c334b187a8c4a6a62eb5 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_500_d9aa9f2aa1576ba26432.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d9aa9f2aa1576ba26432e92adc81967b571b816d643c2bfe558b10ab4422b014 +size 295348 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_500_fccb7da3bff4ab4d12ad.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_500_fccb7da3bff4ab4d12ad.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e6660b5f0a83d92c1d57f6635e3772f0014ba8f6 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_500_fccb7da3bff4ab4d12ad.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fccb7da3bff4ab4d12ada310a5549d23705863bbfc50b5b480932d7eb221f5fa +size 333748 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_600_1467cb3b6f875c7e2639.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_600_1467cb3b6f875c7e2639.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8b65cecd2ff6398dbfebae3bb31a4cb891a64932 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_600_1467cb3b6f875c7e2639.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1467cb3b6f875c7e2639db18607f5744edd8d46c7df00f205f02629dfb1b3a7f +size 209525 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_600_3d60078af4607045df2f.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_600_3d60078af4607045df2f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a76fbc8f85a2c47f5ac37bf72de08236d58e25ff --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_600_3d60078af4607045df2f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d60078af4607045df2fde573d92681d276ecf13737f4d55f1dce8d1bec54f0e +size 265492 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_600_43a0c6b539fdbd257093.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_600_43a0c6b539fdbd257093.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..48c82b34f4f2b1528d8f6de71f9d4baf57992a68 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_600_43a0c6b539fdbd257093.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43a0c6b539fdbd257093a00909db3ea9561b57972c023a5fdf16736a81d6adf1 +size 238544 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_600_43f30c27e3fa9e6c86ec.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_600_43f30c27e3fa9e6c86ec.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b7228f7762e0acf4d7979087013646bef702aabc --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_600_43f30c27e3fa9e6c86ec.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43f30c27e3fa9e6c86ec94aa385f0d8517d441c5b1c8a32fcfc667d57dd40e6a +size 231750 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_600_60df597251e026012d16.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_600_60df597251e026012d16.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..15d5cc0501ba8b699ef5e52497a59dcc8781b2a4 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_600_60df597251e026012d16.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60df597251e026012d16571a4d9dd393abe713a66ed61dcb25d8dde57dee40b5 +size 323737 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_600_7b9509d46936637bcc68.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_600_7b9509d46936637bcc68.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c0ccca2a44b8a8ecaf911833c0981bb9851b22ae --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_600_7b9509d46936637bcc68.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b9509d46936637bcc68ce35408ae86ebcb9ffcdd94777f64bdf74773965a3f6 +size 328614 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_600_82db3c09e17b84575be1.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_600_82db3c09e17b84575be1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..50f87c27b7dddbf49b973d4b477c0cee372ef48f --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_600_82db3c09e17b84575be1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82db3c09e17b84575be133edd3453128544f934e4dcb4d3c5a6dc3c72a1f73f8 +size 291039 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_600_8ab6c8d2a459baf2075f.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_600_8ab6c8d2a459baf2075f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6c5e6730bbd73eb716c668e2aaa33307da7bacab --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_600_8ab6c8d2a459baf2075f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ab6c8d2a459baf2075fea328069fbdd2683d238ea5945ae4482d91374f7c1e5 +size 273314 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_600_9225e2625894ebb01c82.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_600_9225e2625894ebb01c82.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d719a0605abc21e7c0e017bf6dd3bd64210ba87f --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_600_9225e2625894ebb01c82.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9225e2625894ebb01c820b7f79ac39df97bb4aef258070d73a13ed68ff49041c +size 246341 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_600_aab249f793c15d258e05.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_600_aab249f793c15d258e05.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5cb4ef0384fe4fbc8e44cbc73880bc60e83a4123 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/generated_600_aab249f793c15d258e05.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aab249f793c15d258e05c52b58935ca710967675e0183874fb651b175d70f83f +size 223383 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_100_15ac5b765a401d035db1.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_100_15ac5b765a401d035db1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39d693cdae1da9a3a67729fa63439503876eafbc --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_100_15ac5b765a401d035db1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117 +size 190004 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_100_294040ac378ceaa34329.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_100_294040ac378ceaa34329.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23dc14931dfca40fe3f7d6b1fdefe0a1edca1ab9 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_100_294040ac378ceaa34329.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c +size 249372 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_100_6ff046418b5d41477989.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_100_6ff046418b5d41477989.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95ab423dcc64a40477b5d8f2e5d44351e2781109 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_100_6ff046418b5d41477989.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac +size 214001 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_100_80aab00fcfa85d5db98e.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_100_80aab00fcfa85d5db98e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09ef3a36f7d3e07e8b458916212891367b079325 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_100_80aab00fcfa85d5db98e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327 +size 182028 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_100_884b2d197a5511393680.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_100_884b2d197a5511393680.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..550cb7c37290fce2f4554ad8f26113320700bfa3 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_100_884b2d197a5511393680.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214 +size 170922 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_100_a076d17e202ed18ebdf8.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_100_a076d17e202ed18ebdf8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c59919de964bcf66a3cfe53f52bedd0a8b91df2e --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_100_a076d17e202ed18ebdf8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb +size 165516 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_100_c0196720a45889bb2198.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_100_c0196720a45889bb2198.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018bd36397d571746795d1f4619cc8f4fc2766da --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_100_c0196720a45889bb2198.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8 +size 173236 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_100_c0f892b5fba78348af85.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_100_c0f892b5fba78348af85.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f296c5f3aec5b609e307e17bbeffa25264deb91 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_100_c0f892b5fba78348af85.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8 +size 222986 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_100_f5463a4ea6f68a39c5d0.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_100_f5463a4ea6f68a39c5d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..078934eedc27e8de1e5fed0272335f707868e553 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_100_f5463a4ea6f68a39c5d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395 +size 164630 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_100_f7412c9e170ffd299748.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_100_f7412c9e170ffd299748.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eaf7bba0c634cee305854ae2bbd756c90092d38 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_100_f7412c9e170ffd299748.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273 +size 227490 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_200_15ac5b765a401d035db1.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_200_15ac5b765a401d035db1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39d693cdae1da9a3a67729fa63439503876eafbc --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_200_15ac5b765a401d035db1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117 +size 190004 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_200_294040ac378ceaa34329.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_200_294040ac378ceaa34329.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23dc14931dfca40fe3f7d6b1fdefe0a1edca1ab9 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_200_294040ac378ceaa34329.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c +size 249372 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_200_6ff046418b5d41477989.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_200_6ff046418b5d41477989.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95ab423dcc64a40477b5d8f2e5d44351e2781109 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_200_6ff046418b5d41477989.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac +size 214001 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_200_80aab00fcfa85d5db98e.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_200_80aab00fcfa85d5db98e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09ef3a36f7d3e07e8b458916212891367b079325 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_200_80aab00fcfa85d5db98e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327 +size 182028 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_200_884b2d197a5511393680.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_200_884b2d197a5511393680.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..550cb7c37290fce2f4554ad8f26113320700bfa3 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_200_884b2d197a5511393680.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214 +size 170922 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_200_a076d17e202ed18ebdf8.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_200_a076d17e202ed18ebdf8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c59919de964bcf66a3cfe53f52bedd0a8b91df2e --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_200_a076d17e202ed18ebdf8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb +size 165516 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_200_c0196720a45889bb2198.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_200_c0196720a45889bb2198.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018bd36397d571746795d1f4619cc8f4fc2766da --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_200_c0196720a45889bb2198.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8 +size 173236 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_200_c0f892b5fba78348af85.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_200_c0f892b5fba78348af85.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f296c5f3aec5b609e307e17bbeffa25264deb91 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_200_c0f892b5fba78348af85.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8 +size 222986 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_200_f5463a4ea6f68a39c5d0.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_200_f5463a4ea6f68a39c5d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..078934eedc27e8de1e5fed0272335f707868e553 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_200_f5463a4ea6f68a39c5d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395 +size 164630 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_200_f7412c9e170ffd299748.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_200_f7412c9e170ffd299748.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eaf7bba0c634cee305854ae2bbd756c90092d38 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_200_f7412c9e170ffd299748.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273 +size 227490 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_300_15ac5b765a401d035db1.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_300_15ac5b765a401d035db1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39d693cdae1da9a3a67729fa63439503876eafbc --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_300_15ac5b765a401d035db1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117 +size 190004 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_300_294040ac378ceaa34329.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_300_294040ac378ceaa34329.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23dc14931dfca40fe3f7d6b1fdefe0a1edca1ab9 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_300_294040ac378ceaa34329.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c +size 249372 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_300_6ff046418b5d41477989.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_300_6ff046418b5d41477989.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95ab423dcc64a40477b5d8f2e5d44351e2781109 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_300_6ff046418b5d41477989.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac +size 214001 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_300_80aab00fcfa85d5db98e.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_300_80aab00fcfa85d5db98e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09ef3a36f7d3e07e8b458916212891367b079325 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_300_80aab00fcfa85d5db98e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327 +size 182028 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_300_884b2d197a5511393680.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_300_884b2d197a5511393680.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..550cb7c37290fce2f4554ad8f26113320700bfa3 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_300_884b2d197a5511393680.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214 +size 170922 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_300_a076d17e202ed18ebdf8.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_300_a076d17e202ed18ebdf8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c59919de964bcf66a3cfe53f52bedd0a8b91df2e --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_300_a076d17e202ed18ebdf8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb +size 165516 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_300_c0196720a45889bb2198.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_300_c0196720a45889bb2198.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018bd36397d571746795d1f4619cc8f4fc2766da --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_300_c0196720a45889bb2198.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8 +size 173236 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_300_c0f892b5fba78348af85.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_300_c0f892b5fba78348af85.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f296c5f3aec5b609e307e17bbeffa25264deb91 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_300_c0f892b5fba78348af85.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8 +size 222986 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_300_f5463a4ea6f68a39c5d0.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_300_f5463a4ea6f68a39c5d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..078934eedc27e8de1e5fed0272335f707868e553 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_300_f5463a4ea6f68a39c5d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395 +size 164630 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_300_f7412c9e170ffd299748.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_300_f7412c9e170ffd299748.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eaf7bba0c634cee305854ae2bbd756c90092d38 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_300_f7412c9e170ffd299748.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273 +size 227490 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_400_15ac5b765a401d035db1.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_400_15ac5b765a401d035db1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39d693cdae1da9a3a67729fa63439503876eafbc --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_400_15ac5b765a401d035db1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117 +size 190004 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_400_294040ac378ceaa34329.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_400_294040ac378ceaa34329.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23dc14931dfca40fe3f7d6b1fdefe0a1edca1ab9 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_400_294040ac378ceaa34329.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c +size 249372 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_400_6ff046418b5d41477989.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_400_6ff046418b5d41477989.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95ab423dcc64a40477b5d8f2e5d44351e2781109 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_400_6ff046418b5d41477989.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac +size 214001 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_400_80aab00fcfa85d5db98e.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_400_80aab00fcfa85d5db98e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09ef3a36f7d3e07e8b458916212891367b079325 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_400_80aab00fcfa85d5db98e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327 +size 182028 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_400_884b2d197a5511393680.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_400_884b2d197a5511393680.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..550cb7c37290fce2f4554ad8f26113320700bfa3 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_400_884b2d197a5511393680.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214 +size 170922 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_400_a076d17e202ed18ebdf8.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_400_a076d17e202ed18ebdf8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c59919de964bcf66a3cfe53f52bedd0a8b91df2e --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_400_a076d17e202ed18ebdf8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb +size 165516 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_400_c0196720a45889bb2198.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_400_c0196720a45889bb2198.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018bd36397d571746795d1f4619cc8f4fc2766da --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_400_c0196720a45889bb2198.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8 +size 173236 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_400_c0f892b5fba78348af85.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_400_c0f892b5fba78348af85.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f296c5f3aec5b609e307e17bbeffa25264deb91 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_400_c0f892b5fba78348af85.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8 +size 222986 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_400_f5463a4ea6f68a39c5d0.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_400_f5463a4ea6f68a39c5d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..078934eedc27e8de1e5fed0272335f707868e553 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_400_f5463a4ea6f68a39c5d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395 +size 164630 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_400_f7412c9e170ffd299748.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_400_f7412c9e170ffd299748.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eaf7bba0c634cee305854ae2bbd756c90092d38 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_400_f7412c9e170ffd299748.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273 +size 227490 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_500_15ac5b765a401d035db1.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_500_15ac5b765a401d035db1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39d693cdae1da9a3a67729fa63439503876eafbc --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_500_15ac5b765a401d035db1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117 +size 190004 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_500_294040ac378ceaa34329.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_500_294040ac378ceaa34329.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23dc14931dfca40fe3f7d6b1fdefe0a1edca1ab9 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_500_294040ac378ceaa34329.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c +size 249372 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_500_6ff046418b5d41477989.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_500_6ff046418b5d41477989.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95ab423dcc64a40477b5d8f2e5d44351e2781109 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_500_6ff046418b5d41477989.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac +size 214001 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_500_80aab00fcfa85d5db98e.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_500_80aab00fcfa85d5db98e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09ef3a36f7d3e07e8b458916212891367b079325 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_500_80aab00fcfa85d5db98e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327 +size 182028 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_500_884b2d197a5511393680.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_500_884b2d197a5511393680.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..550cb7c37290fce2f4554ad8f26113320700bfa3 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_500_884b2d197a5511393680.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214 +size 170922 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_500_a076d17e202ed18ebdf8.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_500_a076d17e202ed18ebdf8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c59919de964bcf66a3cfe53f52bedd0a8b91df2e --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_500_a076d17e202ed18ebdf8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb +size 165516 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_500_c0196720a45889bb2198.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_500_c0196720a45889bb2198.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018bd36397d571746795d1f4619cc8f4fc2766da --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_500_c0196720a45889bb2198.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8 +size 173236 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_500_c0f892b5fba78348af85.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_500_c0f892b5fba78348af85.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f296c5f3aec5b609e307e17bbeffa25264deb91 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_500_c0f892b5fba78348af85.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8 +size 222986 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_500_f5463a4ea6f68a39c5d0.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_500_f5463a4ea6f68a39c5d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..078934eedc27e8de1e5fed0272335f707868e553 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_500_f5463a4ea6f68a39c5d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395 +size 164630 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_500_f7412c9e170ffd299748.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_500_f7412c9e170ffd299748.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eaf7bba0c634cee305854ae2bbd756c90092d38 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_500_f7412c9e170ffd299748.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273 +size 227490 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_600_15ac5b765a401d035db1.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_600_15ac5b765a401d035db1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39d693cdae1da9a3a67729fa63439503876eafbc --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_600_15ac5b765a401d035db1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117 +size 190004 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_600_294040ac378ceaa34329.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_600_294040ac378ceaa34329.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23dc14931dfca40fe3f7d6b1fdefe0a1edca1ab9 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_600_294040ac378ceaa34329.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c +size 249372 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_600_6ff046418b5d41477989.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_600_6ff046418b5d41477989.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95ab423dcc64a40477b5d8f2e5d44351e2781109 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_600_6ff046418b5d41477989.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac +size 214001 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_600_80aab00fcfa85d5db98e.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_600_80aab00fcfa85d5db98e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09ef3a36f7d3e07e8b458916212891367b079325 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_600_80aab00fcfa85d5db98e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327 +size 182028 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_600_884b2d197a5511393680.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_600_884b2d197a5511393680.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..550cb7c37290fce2f4554ad8f26113320700bfa3 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_600_884b2d197a5511393680.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214 +size 170922 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_600_a076d17e202ed18ebdf8.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_600_a076d17e202ed18ebdf8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c59919de964bcf66a3cfe53f52bedd0a8b91df2e --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_600_a076d17e202ed18ebdf8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb +size 165516 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_600_c0196720a45889bb2198.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_600_c0196720a45889bb2198.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018bd36397d571746795d1f4619cc8f4fc2766da --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_600_c0196720a45889bb2198.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8 +size 173236 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_600_c0f892b5fba78348af85.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_600_c0f892b5fba78348af85.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f296c5f3aec5b609e307e17bbeffa25264deb91 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_600_c0f892b5fba78348af85.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8 +size 222986 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_600_f5463a4ea6f68a39c5d0.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_600_f5463a4ea6f68a39c5d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..078934eedc27e8de1e5fed0272335f707868e553 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_600_f5463a4ea6f68a39c5d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395 +size 164630 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_600_f7412c9e170ffd299748.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_600_f7412c9e170ffd299748.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eaf7bba0c634cee305854ae2bbd756c90092d38 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/media/videos/val0/reconstructed_600_f7412c9e170ffd299748.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273 +size 227490 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/output.log b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..e5cdaebdc61c792566ba26813c4e9e677ab76916 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/output.log @@ -0,0 +1,12486 @@ +[Apr 27, 2026 - 10:02:47 | INFO | fastgen.callbacks.wandb:init_wandb:290 ] Wandb code upload disabled (set WANDB_UPLOAD_CODE=true to enable) +[Apr 27, 2026 - 10:02:47 | SUCCESS | fastgen.trainer:__init__:53 ] Callbacks initialized successfully +[Apr 27, 2026 - 10:02:47 | INFO | fastgen.trainer:__init__:57 ] Callback synchronization complete +[Apr 27, 2026 - 10:02:47 | INFO | fastgen.trainer:__init__:60 ] Initializing checkpointer... +[Apr 27, 2026 - 10:02:47 | SUCCESS | fastgen.trainer:__init__:65 ] Checkpointer initialized successfully +[Apr 27, 2026 - 10:02:47 | SUCCESS | __main__:main:33 ] Trainer initialized successfully +[Apr 27, 2026 - 10:02:47 | INFO | fastgen.trainer:run:77 ] Starting training +[Apr 27, 2026 - 10:02:47 | INFO | fastgen.trainer:run:80 ] Initializing callbacks and model ... +[Apr 27, 2026 - 10:02:47 | INFO | fastgen.utils.checkpointer:load:151 ] Loading model from /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_t769_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth +[Apr 27, 2026 - 10:03:20 | INFO | fastgen.utils.checkpointer:load:154 ] Loading the model_dict... +[Apr 27, 2026 - 10:03:32 | INFO | fastgen.utils.checkpointer:load:159 ] Model net, loading info: +[Apr 27, 2026 - 10:03:32 | INFO | fastgen.trainer:load_pretrained_ckpt:252 ] Loaded net model from net in /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_t769_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth at iteration 5000 +[Apr 27, 2026 - 10:03:32 | INFO | fastgen.trainer:load_pretrained_ckpt:262 ] Setting resume_iter for model to 5000. +[Apr 27, 2026 - 10:03:32 | INFO | fastgen.trainer:run:95 ] Starting model.on_train_begin ... +[Apr 27, 2026 - 10:04:41 | INFO | fastgen.methods.model:on_train_begin:296 ] Teacher check: add_teacher_to_fsdp_dict=True, fsdp_dict keys=['net', 'fake_score', 'teacher'], teacher in fsdp_dict=True +[Apr 27, 2026 - 10:04:41 | INFO | fastgen.trainer:run:99 ] model.on_train_begin completed +[Apr 27, 2026 - 10:04:41 | INFO | fastgen.trainer:run:110 ] Wrapping model into fsdp .. +[Apr 27, 2026 - 10:04:41 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:102 ] Fully sharding model with 4 ranks... +[Apr 27, 2026 - 10:04:41 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'net' (1.42B params)... +[Apr 27, 2026 - 10:04:41 | INFO | fastgen.networks.OmniAvatar.network_causal:fully_shard:2166 ] CausalOmniAvatarWan: keeping manual gradient checkpointing (not using apply_fsdp_checkpointing due to KV cache dynamics) +[Apr 27, 2026 - 10:04:44 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[Apr 27, 2026 - 10:04:45 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped net in 3.1s +[Apr 27, 2026 - 10:04:45 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'fake_score' (1.60B params)... +[Apr 27, 2026 - 10:04:45 | INFO | fastgen.networks.OmniAvatar.network:fully_shard:774 ] OmniAvatarWan: keeping manual gradient checkpointing (checkpoint_wrapper incompatible with inter-block audio injection) +[Apr 27, 2026 - 10:04:47 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[Apr 27, 2026 - 10:04:47 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped fake_score in 2.8s +[Apr 27, 2026 - 10:04:47 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'teacher' (14.29B params)... +[Apr 27, 2026 - 10:04:47 | INFO | fastgen.networks.OmniAvatar.network:fully_shard:774 ] OmniAvatarWan: keeping manual gradient checkpointing (checkpoint_wrapper incompatible with inter-block audio injection) +[Apr 27, 2026 - 10:05:01 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[Apr 27, 2026 - 10:05:01 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped teacher in 13.8s +[Apr 27, 2026 - 10:05:01 | INFO | fastgen.trainer:run:118 ] FSDP wrapping completed +[Apr 27, 2026 - 10:05:01 | INFO | fastgen.callbacks.ema:on_model_init_end:64 ] EMA ema is not enabled, skipping callback. +[Apr 27, 2026 - 10:05:01 | INFO | fastgen.trainer:run:133 ] Auto-Resume Details: None +[Apr 27, 2026 - 10:05:01 | INFO | fastgen.utils.basic_utils:set_random_seed:144 ] Using random seed 0. +[Apr 27, 2026 - 10:05:01 | INFO | fastgen.trainer:run:165 ] Instantiating dataloader... +[Apr 27, 2026 - 10:05:09 | INFO | fastgen.callbacks.wandb:on_dataloader_init_end:422 ] SKIP_GT_VAL_UPLOAD=1 — skipping GT val video upload +[Apr 27, 2026 - 10:05:09 | INFO | fastgen.callbacks.gpu_stats:on_train_begin:57 ] every_n to measure gpus stats: 1 +[Apr 27, 2026 - 10:05:09 | INFO | fastgen.callbacks.train_profiler:on_train_begin:54 ] every_n to profile trainer: 1 +[Apr 27, 2026 - 10:05:09 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] model (OmniAvatarSelfForcingReDMD) has 1596.36 M trainable and 17325.46 M total params (logical). +[Apr 27, 2026 - 10:05:09 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 420.42 M trainable and 4364.06 M total params LOCAL on rank 0. +[Apr 27, 2026 - 10:05:09 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 420.14 M trainable and 4362.26 M total params LOCAL on rank 1. +[Apr 27, 2026 - 10:05:09 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 420.14 M trainable and 4362.26 M total params LOCAL on rank 2. +[Apr 27, 2026 - 10:05:09 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 420.14 M trainable and 4362.26 M total params LOCAL on rank 3. +[Apr 27, 2026 - 10:05:09 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] net (CausalOmniAvatarWan) has 1421.38 M trainable and 1421.38 M total params (logical). +[Apr 27, 2026 - 10:05:09 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 376.68 M trainable and 376.68 M total params LOCAL on rank 0. +[Apr 27, 2026 - 10:05:09 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 376.40 M trainable and 376.40 M total params LOCAL on rank 1. +[Apr 27, 2026 - 10:05:09 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 376.40 M trainable and 376.40 M total params LOCAL on rank 2. +[Apr 27, 2026 - 10:05:09 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 376.40 M trainable and 376.40 M total params LOCAL on rank 3. +[Apr 27, 2026 - 10:05:09 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] fake_score (OmniAvatarWan) has 174.98 M trainable and 1596.36 M total params (logical). +[Apr 27, 2026 - 10:05:09 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 43.75 M trainable and 399.30 M total params LOCAL on rank 0. +[Apr 27, 2026 - 10:05:09 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 43.75 M trainable and 399.02 M total params LOCAL on rank 1. +[Apr 27, 2026 - 10:05:09 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 43.75 M trainable and 399.02 M total params LOCAL on rank 2. +[Apr 27, 2026 - 10:05:09 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 43.75 M trainable and 399.02 M total params LOCAL on rank 3. +[Apr 27, 2026 - 10:05:09 | INFO | fastgen.trainer:run:174 ] iter_start: 0 +[Apr 27, 2026 - 10:06:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.565402 38.639732 38.488865 +peak_gpu_mem_gb 60.567137 60.567137 60.567137 +peak_gpu_mem_reserved_gb 62.595703 62.595703 62.595703 +util 90.500000 99.000000 85.000000 +[Apr 27, 2026 - 10:06:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 1 : data loading time 1.39 +[Apr 27, 2026 - 10:06:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 1 : avg forward pass time 11.07 +[Apr 27, 2026 - 10:06:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 1 : backward pass time 12.96 +[Apr 27, 2026 - 10:06:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 1 : optimizer step time 1.25 +[Apr 27, 2026 - 10:06:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 1-------------------- +[Apr 27, 2026 - 10:06:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[Apr 27, 2026 - 10:06:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[Apr 27, 2026 - 10:06:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 10:06:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 10:06:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.741200 38.815273 38.664471 +peak_gpu_mem_gb 62.354346 62.354346 62.354346 +peak_gpu_mem_reserved_gb 64.042969 64.042969 64.042969 +util 96.500000 99.000000 92.000000 +[Apr 27, 2026 - 10:06:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 2 : avg iteration time 49.85 seconds +[Apr 27, 2026 - 10:06:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 2 : data loading time 0.00 +[Apr 27, 2026 - 10:06:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 2 : avg forward pass time 12.85 +[Apr 27, 2026 - 10:06:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 2 : backward pass time 11.59 +[Apr 27, 2026 - 10:06:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 2 : optimizer step time 0.97 +[Apr 27, 2026 - 10:06:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 2-------------------- +[Apr 27, 2026 - 10:06:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[Apr 27, 2026 - 10:06:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[Apr 27, 2026 - 10:06:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 10:06:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 10:07:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.741269 38.815357 38.664547 +peak_gpu_mem_gb 62.354346 62.354346 62.354346 +peak_gpu_mem_reserved_gb 64.042969 64.042969 64.042969 +util 95.000000 99.000000 91.000000 +[Apr 27, 2026 - 10:07:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 3 : avg iteration time 53.50 seconds +[Apr 27, 2026 - 10:07:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 3 : data loading time 0.00 +[Apr 27, 2026 - 10:07:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 3 : avg forward pass time 14.12 +[Apr 27, 2026 - 10:07:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 3 : backward pass time 11.54 +[Apr 27, 2026 - 10:07:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 3 : optimizer step time 0.97 +[Apr 27, 2026 - 10:07:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 3-------------------- +[Apr 27, 2026 - 10:07:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0076 iter count: 1.0 +[Apr 27, 2026 - 10:07:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0076 iter count: 1.0 +[Apr 27, 2026 - 10:07:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 10:07:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 10:08:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.741469 38.815392 38.664585 +peak_gpu_mem_gb 62.354346 62.354346 62.354346 +peak_gpu_mem_reserved_gb 64.077148 64.121094 64.062500 +util 96.750000 99.000000 92.000000 +[Apr 27, 2026 - 10:08:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 4 : avg iteration time 48.31 seconds +[Apr 27, 2026 - 10:08:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 4 : data loading time 0.00 +[Apr 27, 2026 - 10:08:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 4 : avg forward pass time 11.60 +[Apr 27, 2026 - 10:08:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 4 : backward pass time 11.57 +[Apr 27, 2026 - 10:08:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 4 : optimizer step time 0.97 +[Apr 27, 2026 - 10:08:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 4-------------------- +[Apr 27, 2026 - 10:08:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[Apr 27, 2026 - 10:08:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[Apr 27, 2026 - 10:08:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 10:08:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 10:11:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.804677 38.879646 38.725185 +peak_gpu_mem_gb 126.559253 126.559253 126.559253 +peak_gpu_mem_reserved_gb 136.197754 138.166016 130.410156 +util 63.000000 70.000000 58.000000 +[Apr 27, 2026 - 10:11:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 5 : avg iteration time 180.18 seconds +[Apr 27, 2026 - 10:11:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 5 : data loading time 0.00 +[Apr 27, 2026 - 10:11:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 5 : avg forward pass time 80.78 +[Apr 27, 2026 - 10:11:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 5 : backward pass time 9.57 +[Apr 27, 2026 - 10:11:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 5 : optimizer step time 0.42 +[Apr 27, 2026 - 10:11:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 5-------------------- +[Apr 27, 2026 - 10:11:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 42358.2883 iter count: 1.0 +[Apr 27, 2026 - 10:11:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.8022 iter count: 1.0 +[Apr 27, 2026 - 10:11:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 42358.2883iter count: 1.0 +[Apr 27, 2026 - 10:11:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 10:11:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 3.3609iter count: 1.0 +[Apr 27, 2026 - 10:11:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.8474 iter count: 1.0 +[Apr 27, 2026 - 10:11:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.0039 iter count: 1.0 +[Apr 27, 2026 - 10:11:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 53579.0117iter count: 1.0 +[Apr 27, 2026 - 10:11:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 5.4455 iter count: 1.0 +[Apr 27, 2026 - 10:11:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 1212037.6250iter count: 1.0 +[Apr 27, 2026 - 10:11:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.8022iter count: 1.0 +[Apr 27, 2026 - 10:11:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.6668 iter count: 1.0 +[Apr 27, 2026 - 10:11:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 11312.4824iter count: 1.0 +[Apr 27, 2026 - 10:11:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 2.7134 iter count: 1.0 +[Apr 27, 2026 - 10:11:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 227.4311iter count: 1.0 +[Apr 27, 2026 - 10:11:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 6.1654 iter count: 1.0 +[Apr 27, 2026 - 10:11:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 226578.0938iter count: 1.0 +[Apr 27, 2026 - 10:11:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 1.8156 iter count: 1.0 +[Apr 27, 2026 - 10:11:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 37.7612 iter count: 1.0 +[Apr 27, 2026 - 10:11:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[Apr 27, 2026 - 10:11:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 10:11:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 10:12:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.804766 38.879665 38.725201 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.325684 67.437500 67.007812 +util 98.500000 99.000000 97.000000 +[Apr 27, 2026 - 10:12:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 6 : avg iteration time 52.89 seconds +[Apr 27, 2026 - 10:12:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 6 : data loading time 0.00 +[Apr 27, 2026 - 10:12:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 6 : avg forward pass time 14.36 +[Apr 27, 2026 - 10:12:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 6 : backward pass time 11.62 +[Apr 27, 2026 - 10:12:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 6 : optimizer step time 0.97 +[Apr 27, 2026 - 10:12:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 6-------------------- +[Apr 27, 2026 - 10:12:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0083 iter count: 1.0 +[Apr 27, 2026 - 10:12:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[Apr 27, 2026 - 10:12:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 10:12:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 10:13:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.805001 38.879681 38.726116 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.315918 67.419922 67.046875 +util 97.500000 100.000000 92.000000 +[Apr 27, 2026 - 10:13:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 7 : avg iteration time 53.73 seconds +[Apr 27, 2026 - 10:13:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 7 : data loading time 0.00 +[Apr 27, 2026 - 10:13:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 7 : avg forward pass time 14.18 +[Apr 27, 2026 - 10:13:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 7 : backward pass time 12.22 +[Apr 27, 2026 - 10:13:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 7 : optimizer step time 1.57 +[Apr 27, 2026 - 10:13:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 7-------------------- +[Apr 27, 2026 - 10:13:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0080 iter count: 1.0 +[Apr 27, 2026 - 10:13:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0080 iter count: 1.0 +[Apr 27, 2026 - 10:13:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 10:13:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 10:14:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.805011 38.879692 38.726120 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.149902 67.320312 66.931641 +util 98.500000 100.000000 95.000000 +[Apr 27, 2026 - 10:14:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 8 : avg iteration time 52.64 seconds +[Apr 27, 2026 - 10:14:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 8 : data loading time 0.00 +[Apr 27, 2026 - 10:14:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 8 : avg forward pass time 14.24 +[Apr 27, 2026 - 10:14:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 8 : backward pass time 11.61 +[Apr 27, 2026 - 10:14:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 8 : optimizer step time 0.99 +[Apr 27, 2026 - 10:14:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 8-------------------- +[Apr 27, 2026 - 10:14:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0075 iter count: 1.0 +[Apr 27, 2026 - 10:14:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0075 iter count: 1.0 +[Apr 27, 2026 - 10:14:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 10:14:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 10:14:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.805027 38.879723 38.726139 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.184082 67.378906 66.931641 +util 98.250000 100.000000 95.000000 +[Apr 27, 2026 - 10:14:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 9 : avg iteration time 47.40 seconds +[Apr 27, 2026 - 10:14:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 9 : data loading time 0.00 +[Apr 27, 2026 - 10:14:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 9 : avg forward pass time 11.45 +[Apr 27, 2026 - 10:14:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 9 : backward pass time 11.54 +[Apr 27, 2026 - 10:14:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 9 : optimizer step time 0.98 +[Apr 27, 2026 - 10:14:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 9-------------------- +[Apr 27, 2026 - 10:14:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0077 iter count: 1.0 +[Apr 27, 2026 - 10:14:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0077 iter count: 1.0 +[Apr 27, 2026 - 10:14:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 10:14:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 10:17:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.805479 38.880348 38.726788 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.152832 133.191406 133.074219 +util 77.250000 93.000000 65.000000 +[Apr 27, 2026 - 10:17:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 10 : avg iteration time 176.66 seconds +[Apr 27, 2026 - 10:17:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 10 : data loading time 0.00 +[Apr 27, 2026 - 10:17:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 10 : avg forward pass time 78.96 +[Apr 27, 2026 - 10:17:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 10 : backward pass time 9.26 +[Apr 27, 2026 - 10:17:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 10 : optimizer step time 0.34 +[Apr 27, 2026 - 10:17:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 10-------------------- +[Apr 27, 2026 - 10:17:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 25834.0012 iter count: 1.0 +[Apr 27, 2026 - 10:17:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7924 iter count: 1.0 +[Apr 27, 2026 - 10:17:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 25834.0012iter count: 1.0 +[Apr 27, 2026 - 10:17:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 10:17:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 3.9231iter count: 1.0 +[Apr 27, 2026 - 10:17:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.6193 iter count: 1.0 +[Apr 27, 2026 - 10:17:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 6.5540 iter count: 1.0 +[Apr 27, 2026 - 10:17:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 30728.2227iter count: 1.0 +[Apr 27, 2026 - 10:17:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 3.4509 iter count: 1.0 +[Apr 27, 2026 - 10:17:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 492877.5312iter count: 1.0 +[Apr 27, 2026 - 10:17:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7924iter count: 1.0 +[Apr 27, 2026 - 10:17:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 6.5540 iter count: 1.0 +[Apr 27, 2026 - 10:17:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 492877.5312iter count: 1.0 +[Apr 27, 2026 - 10:17:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 2.4193 iter count: 1.0 +[Apr 27, 2026 - 10:17:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 126.2970iter count: 1.0 +[Apr 27, 2026 - 10:17:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.1807 iter count: 1.0 +[Apr 27, 2026 - 10:17:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 4278.9365iter count: 1.0 +[Apr 27, 2026 - 10:17:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.8080 iter count: 1.0 +[Apr 27, 2026 - 10:17:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 110861.9688iter count: 1.0 +[Apr 27, 2026 - 10:17:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0080 iter count: 1.0 +[Apr 27, 2026 - 10:17:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 10:17:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 10:18:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.805506 38.880379 38.726841 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.350098 67.361328 67.339844 +util 96.000000 99.000000 89.000000 +[Apr 27, 2026 - 10:18:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 11 : avg iteration time 51.77 seconds +[Apr 27, 2026 - 10:18:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 11 : data loading time 0.00 +[Apr 27, 2026 - 10:18:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 11 : avg forward pass time 13.77 +[Apr 27, 2026 - 10:18:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 11 : backward pass time 11.70 +[Apr 27, 2026 - 10:18:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 11 : optimizer step time 0.99 +[Apr 27, 2026 - 10:18:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 11-------------------- +[Apr 27, 2026 - 10:18:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0084 iter count: 1.0 +[Apr 27, 2026 - 10:18:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[Apr 27, 2026 - 10:18:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 10:18:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 10:19:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.805506 38.880379 38.726826 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.208496 67.437500 67.068359 +util 99.000000 100.000000 97.000000 +[Apr 27, 2026 - 10:19:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 12 : avg iteration time 52.99 seconds +[Apr 27, 2026 - 10:19:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 12 : data loading time 0.00 +[Apr 27, 2026 - 10:19:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 12 : avg forward pass time 14.15 +[Apr 27, 2026 - 10:19:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 12 : backward pass time 11.55 +[Apr 27, 2026 - 10:19:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 12 : optimizer step time 0.98 +[Apr 27, 2026 - 10:19:40 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 12-------------------- +[Apr 27, 2026 - 10:19:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0073 iter count: 1.0 +[Apr 27, 2026 - 10:19:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0073 iter count: 1.0 +[Apr 27, 2026 - 10:19:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 10:19:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 10:20:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.805749 38.880367 38.726826 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.291504 67.437500 67.048828 +util 94.500000 100.000000 90.000000 +[Apr 27, 2026 - 10:20:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 13 : avg iteration time 54.05 seconds +[Apr 27, 2026 - 10:20:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 13 : data loading time 0.00 +[Apr 27, 2026 - 10:20:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 13 : avg forward pass time 14.22 +[Apr 27, 2026 - 10:20:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 13 : backward pass time 12.52 +[Apr 27, 2026 - 10:20:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 13 : optimizer step time 1.53 +[Apr 27, 2026 - 10:20:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 13-------------------- +[Apr 27, 2026 - 10:20:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0070 iter count: 1.0 +[Apr 27, 2026 - 10:20:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0070 iter count: 1.0 +[Apr 27, 2026 - 10:20:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 10:20:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 10:21:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.806035 38.880371 38.726826 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 68.775879 70.269531 67.126953 +util 94.750000 100.000000 90.000000 +[Apr 27, 2026 - 10:21:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 14 : avg iteration time 50.16 seconds +[Apr 27, 2026 - 10:21:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 14 : data loading time 0.00 +[Apr 27, 2026 - 10:21:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 14 : avg forward pass time 12.73 +[Apr 27, 2026 - 10:21:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 14 : backward pass time 12.13 +[Apr 27, 2026 - 10:21:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 14 : optimizer step time 0.98 +[Apr 27, 2026 - 10:21:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 14-------------------- +[Apr 27, 2026 - 10:21:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0077 iter count: 1.0 +[Apr 27, 2026 - 10:21:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0077 iter count: 1.0 +[Apr 27, 2026 - 10:21:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 10:21:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 10:24:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.806865 38.881500 38.727707 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.144531 133.193359 133.056641 +util 69.000000 76.000000 64.000000 +[Apr 27, 2026 - 10:24:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 15 : avg iteration time 170.48 seconds +[Apr 27, 2026 - 10:24:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 15 : data loading time 0.00 +[Apr 27, 2026 - 10:24:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 15 : avg forward pass time 76.42 +[Apr 27, 2026 - 10:24:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 15 : backward pass time 8.40 +[Apr 27, 2026 - 10:24:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 15 : optimizer step time 0.33 +[Apr 27, 2026 - 10:24:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 15-------------------- +[Apr 27, 2026 - 10:24:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 40159.8519 iter count: 1.0 +[Apr 27, 2026 - 10:24:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7805 iter count: 1.0 +[Apr 27, 2026 - 10:24:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 40159.8519iter count: 1.0 +[Apr 27, 2026 - 10:24:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 10:24:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 3.3572iter count: 1.0 +[Apr 27, 2026 - 10:24:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.5704 iter count: 1.0 +[Apr 27, 2026 - 10:24:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 6.9872 iter count: 1.0 +[Apr 27, 2026 - 10:24:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 47439.1211iter count: 1.0 +[Apr 27, 2026 - 10:24:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 3.1290 iter count: 1.0 +[Apr 27, 2026 - 10:24:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 1172213.3750iter count: 1.0 +[Apr 27, 2026 - 10:24:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7805iter count: 1.0 +[Apr 27, 2026 - 10:24:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.7721 iter count: 1.0 +[Apr 27, 2026 - 10:24:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 1889.5646iter count: 1.0 +[Apr 27, 2026 - 10:24:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.4215 iter count: 1.0 +[Apr 27, 2026 - 10:24:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 6925.8867iter count: 1.0 +[Apr 27, 2026 - 10:24:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 2.6207 iter count: 1.0 +[Apr 27, 2026 - 10:24:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 188.9212iter count: 1.0 +[Apr 27, 2026 - 10:24:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 2.5154 iter count: 1.0 +[Apr 27, 2026 - 10:24:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 153.0545iter count: 1.0 +[Apr 27, 2026 - 10:24:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0069 iter count: 1.0 +[Apr 27, 2026 - 10:24:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 10:24:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 10:25:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.807540 38.881504 38.728603 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 66.993652 67.027344 66.968750 +util 98.500000 100.000000 96.000000 +[Apr 27, 2026 - 10:25:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 16 : avg iteration time 49.09 seconds +[Apr 27, 2026 - 10:25:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 16 : data loading time 0.00 +[Apr 27, 2026 - 10:25:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 16 : avg forward pass time 11.83 +[Apr 27, 2026 - 10:25:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 16 : backward pass time 11.57 +[Apr 27, 2026 - 10:25:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 16 : optimizer step time 0.98 +[Apr 27, 2026 - 10:25:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 16-------------------- +[Apr 27, 2026 - 10:25:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0069 iter count: 1.0 +[Apr 27, 2026 - 10:25:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0069 iter count: 1.0 +[Apr 27, 2026 - 10:25:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 10:25:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 10:25:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.807768 38.882431 38.728619 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 68.517090 69.820312 67.007812 +util 98.250000 100.000000 94.000000 +[Apr 27, 2026 - 10:25:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 17 : avg iteration time 50.98 seconds +[Apr 27, 2026 - 10:25:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 17 : data loading time 0.00 +[Apr 27, 2026 - 10:25:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 17 : avg forward pass time 12.78 +[Apr 27, 2026 - 10:25:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 17 : backward pass time 12.22 +[Apr 27, 2026 - 10:25:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 17 : optimizer step time 1.59 +[Apr 27, 2026 - 10:25:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 17-------------------- +[Apr 27, 2026 - 10:25:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0073 iter count: 1.0 +[Apr 27, 2026 - 10:25:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0073 iter count: 1.0 +[Apr 27, 2026 - 10:25:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 10:25:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 10:26:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.807979 38.883308 38.728573 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.130371 67.458984 66.988281 +util 98.250000 99.000000 96.000000 +[Apr 27, 2026 - 10:26:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 18 : avg iteration time 50.23 seconds +[Apr 27, 2026 - 10:26:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 18 : data loading time 0.00 +[Apr 27, 2026 - 10:26:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 18 : avg forward pass time 12.70 +[Apr 27, 2026 - 10:26:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 18 : backward pass time 12.24 +[Apr 27, 2026 - 10:26:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 18 : optimizer step time 1.00 +[Apr 27, 2026 - 10:26:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 18-------------------- +[Apr 27, 2026 - 10:26:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0072 iter count: 1.0 +[Apr 27, 2026 - 10:26:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0072 iter count: 1.0 +[Apr 27, 2026 - 10:26:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 10:26:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 10:27:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.807970 38.883308 38.728596 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.101074 67.439453 66.968750 +util 94.000000 99.000000 90.000000 +[Apr 27, 2026 - 10:27:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 19 : avg iteration time 49.66 seconds +[Apr 27, 2026 - 10:27:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 19 : data loading time 0.00 +[Apr 27, 2026 - 10:27:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 19 : avg forward pass time 12.71 +[Apr 27, 2026 - 10:27:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 19 : backward pass time 11.57 +[Apr 27, 2026 - 10:27:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 19 : optimizer step time 0.99 +[Apr 27, 2026 - 10:27:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 19-------------------- +[Apr 27, 2026 - 10:27:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0072 iter count: 1.0 +[Apr 27, 2026 - 10:27:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0072 iter count: 1.0 +[Apr 27, 2026 - 10:27:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 10:27:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 10:30:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.807755 38.883358 38.728668 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.120605 133.193359 133.076172 +util 77.250000 90.000000 68.000000 +[Apr 27, 2026 - 10:30:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 20 : avg iteration time 177.93 seconds +[Apr 27, 2026 - 10:30:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 20 : data loading time 0.00 +[Apr 27, 2026 - 10:30:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 20 : avg forward pass time 79.83 +[Apr 27, 2026 - 10:30:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 20 : backward pass time 9.63 +[Apr 27, 2026 - 10:30:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 20 : optimizer step time 0.34 +[Apr 27, 2026 - 10:30:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 20-------------------- +[Apr 27, 2026 - 10:30:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 42551.5172 iter count: 1.0 +[Apr 27, 2026 - 10:30:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.8619 iter count: 1.0 +[Apr 27, 2026 - 10:30:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 42551.5172iter count: 1.0 +[Apr 27, 2026 - 10:30:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 10:30:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 3.5861iter count: 1.0 +[Apr 27, 2026 - 10:30:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.2312 iter count: 1.0 +[Apr 27, 2026 - 10:30:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 6.8158 iter count: 1.0 +[Apr 27, 2026 - 10:30:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 53699.7812iter count: 1.0 +[Apr 27, 2026 - 10:30:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 11.7318iter count: 1.0 +[Apr 27, 2026 - 10:30:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 831964.5000iter count: 1.0 +[Apr 27, 2026 - 10:30:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.8619iter count: 1.0 +[Apr 27, 2026 - 10:30:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.3084 iter count: 1.0 +[Apr 27, 2026 - 10:30:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 40813.2539iter count: 1.0 +[Apr 27, 2026 - 10:30:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 2.0441 iter count: 1.0 +[Apr 27, 2026 - 10:30:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 59.6381 iter count: 1.0 +[Apr 27, 2026 - 10:30:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.4626 iter count: 1.0 +[Apr 27, 2026 - 10:30:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 7519.7573iter count: 1.0 +[Apr 27, 2026 - 10:30:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.8158 iter count: 1.0 +[Apr 27, 2026 - 10:30:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 831964.5000iter count: 1.0 +[Apr 27, 2026 - 10:30:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0075 iter count: 1.0 +[Apr 27, 2026 - 10:30:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 10:30:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 10:31:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.807869 38.883347 38.728668 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.027832 67.144531 66.951172 +util 94.750000 99.000000 91.000000 +[Apr 27, 2026 - 10:31:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 21 : avg iteration time 49.48 seconds +[Apr 27, 2026 - 10:31:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 21 : data loading time 0.00 +[Apr 27, 2026 - 10:31:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 21 : avg forward pass time 12.24 +[Apr 27, 2026 - 10:31:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 21 : backward pass time 12.42 +[Apr 27, 2026 - 10:31:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 21 : optimizer step time 1.02 +[Apr 27, 2026 - 10:31:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 21-------------------- +[Apr 27, 2026 - 10:31:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0064 iter count: 1.0 +[Apr 27, 2026 - 10:31:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0064 iter count: 1.0 +[Apr 27, 2026 - 10:31:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 10:31:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 10:32:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808113 38.883347 38.728672 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.179199 67.380859 66.988281 +util 95.500000 100.000000 91.000000 +[Apr 27, 2026 - 10:32:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 22 : avg iteration time 48.96 seconds +[Apr 27, 2026 - 10:32:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 22 : data loading time 0.00 +[Apr 27, 2026 - 10:32:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 22 : avg forward pass time 12.43 +[Apr 27, 2026 - 10:32:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 22 : backward pass time 11.55 +[Apr 27, 2026 - 10:32:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 22 : optimizer step time 0.99 +[Apr 27, 2026 - 10:32:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 22-------------------- +[Apr 27, 2026 - 10:32:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0072 iter count: 1.0 +[Apr 27, 2026 - 10:32:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0072 iter count: 1.0 +[Apr 27, 2026 - 10:32:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 10:32:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 10:33:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808364 38.883354 38.728672 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.076660 67.378906 66.951172 +util 97.750000 100.000000 94.000000 +[Apr 27, 2026 - 10:33:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 23 : avg iteration time 50.27 seconds +[Apr 27, 2026 - 10:33:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 23 : data loading time 0.00 +[Apr 27, 2026 - 10:33:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 23 : avg forward pass time 12.71 +[Apr 27, 2026 - 10:33:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 23 : backward pass time 11.60 +[Apr 27, 2026 - 10:33:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 23 : optimizer step time 1.39 +[Apr 27, 2026 - 10:33:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 23-------------------- +[Apr 27, 2026 - 10:33:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0067 iter count: 1.0 +[Apr 27, 2026 - 10:33:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0067 iter count: 1.0 +[Apr 27, 2026 - 10:33:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 10:33:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 10:33:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808363 38.883350 38.728676 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.130371 67.400391 66.988281 +util 97.500000 100.000000 94.000000 +[Apr 27, 2026 - 10:33:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 24 : avg iteration time 50.20 seconds +[Apr 27, 2026 - 10:33:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 24 : data loading time 0.00 +[Apr 27, 2026 - 10:33:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 24 : avg forward pass time 12.57 +[Apr 27, 2026 - 10:33:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 24 : backward pass time 12.49 +[Apr 27, 2026 - 10:33:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 24 : optimizer step time 1.00 +[Apr 27, 2026 - 10:33:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 24-------------------- +[Apr 27, 2026 - 10:33:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0073 iter count: 1.0 +[Apr 27, 2026 - 10:33:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0073 iter count: 1.0 +[Apr 27, 2026 - 10:33:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 10:33:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 10:36:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.807751 38.883358 38.728680 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.182129 133.193359 133.152344 +util 71.250000 78.000000 67.000000 +[Apr 27, 2026 - 10:36:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 25 : avg iteration time 170.46 seconds +[Apr 27, 2026 - 10:36:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 25 : data loading time 0.00 +[Apr 27, 2026 - 10:36:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 25 : avg forward pass time 76.40 +[Apr 27, 2026 - 10:36:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 25 : backward pass time 8.43 +[Apr 27, 2026 - 10:36:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 25 : optimizer step time 0.37 +[Apr 27, 2026 - 10:36:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 25-------------------- +[Apr 27, 2026 - 10:36:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 49704.6593 iter count: 1.0 +[Apr 27, 2026 - 10:36:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.8212 iter count: 1.0 +[Apr 27, 2026 - 10:36:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 49704.6593iter count: 1.0 +[Apr 27, 2026 - 10:36:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 10:36:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 3.2096iter count: 1.0 +[Apr 27, 2026 - 10:36:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.5387 iter count: 1.0 +[Apr 27, 2026 - 10:36:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 6.9678 iter count: 1.0 +[Apr 27, 2026 - 10:36:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 59966.9648iter count: 1.0 +[Apr 27, 2026 - 10:36:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 2.9370 iter count: 1.0 +[Apr 27, 2026 - 10:36:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 1127706.6250iter count: 1.0 +[Apr 27, 2026 - 10:36:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.8212iter count: 1.0 +[Apr 27, 2026 - 10:36:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 1.8808 iter count: 1.0 +[Apr 27, 2026 - 10:36:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 43.0162 iter count: 1.0 +[Apr 27, 2026 - 10:36:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 0.8183 iter count: 1.0 +[Apr 27, 2026 - 10:36:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 5.1379 iter count: 1.0 +[Apr 27, 2026 - 10:36:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.6707 iter count: 1.0 +[Apr 27, 2026 - 10:36:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 1542.8062iter count: 1.0 +[Apr 27, 2026 - 10:36:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.0137 iter count: 1.0 +[Apr 27, 2026 - 10:36:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 167261.7344iter count: 1.0 +[Apr 27, 2026 - 10:36:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0066 iter count: 1.0 +[Apr 27, 2026 - 10:36:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 10:36:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 10:37:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.807822 38.883358 38.728680 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.364746 67.515625 66.988281 +util 96.500000 99.000000 90.000000 +[Apr 27, 2026 - 10:37:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 26 : avg iteration time 49.25 seconds +[Apr 27, 2026 - 10:37:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 26 : data loading time 0.00 +[Apr 27, 2026 - 10:37:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 26 : avg forward pass time 11.95 +[Apr 27, 2026 - 10:37:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 26 : backward pass time 11.54 +[Apr 27, 2026 - 10:37:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 26 : optimizer step time 0.99 +[Apr 27, 2026 - 10:37:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 26-------------------- +[Apr 27, 2026 - 10:37:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0076 iter count: 1.0 +[Apr 27, 2026 - 10:37:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0076 iter count: 1.0 +[Apr 27, 2026 - 10:37:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 10:37:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 10:38:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808193 38.883362 38.728703 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.306152 67.439453 67.027344 +util 97.500000 100.000000 92.000000 +[Apr 27, 2026 - 10:38:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 27 : avg iteration time 53.95 seconds +[Apr 27, 2026 - 10:38:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 27 : data loading time 0.00 +[Apr 27, 2026 - 10:38:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 27 : avg forward pass time 14.17 +[Apr 27, 2026 - 10:38:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 27 : backward pass time 12.57 +[Apr 27, 2026 - 10:38:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 27 : optimizer step time 1.47 +[Apr 27, 2026 - 10:38:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 27-------------------- +[Apr 27, 2026 - 10:38:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0072 iter count: 1.0 +[Apr 27, 2026 - 10:38:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0072 iter count: 1.0 +[Apr 27, 2026 - 10:38:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 10:38:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 10:39:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808188 38.883362 38.728680 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.223145 67.458984 67.027344 +util 98.000000 100.000000 94.000000 +[Apr 27, 2026 - 10:39:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 28 : avg iteration time 52.80 seconds +[Apr 27, 2026 - 10:39:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 28 : data loading time 0.00 +[Apr 27, 2026 - 10:39:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 28 : avg forward pass time 14.35 +[Apr 27, 2026 - 10:39:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 28 : backward pass time 11.57 +[Apr 27, 2026 - 10:39:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 28 : optimizer step time 0.99 +[Apr 27, 2026 - 10:39:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 28-------------------- +[Apr 27, 2026 - 10:39:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0066 iter count: 1.0 +[Apr 27, 2026 - 10:39:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0066 iter count: 1.0 +[Apr 27, 2026 - 10:39:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 10:39:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 10:40:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808343 38.883366 38.728683 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 68.619629 70.210938 67.027344 +util 97.500000 99.000000 93.000000 +[Apr 27, 2026 - 10:40:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 29 : avg iteration time 50.58 seconds +[Apr 27, 2026 - 10:40:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 29 : data loading time 0.00 +[Apr 27, 2026 - 10:40:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 29 : avg forward pass time 12.63 +[Apr 27, 2026 - 10:40:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 29 : backward pass time 11.57 +[Apr 27, 2026 - 10:40:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 29 : optimizer step time 0.97 +[Apr 27, 2026 - 10:40:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 29-------------------- +[Apr 27, 2026 - 10:40:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0067 iter count: 1.0 +[Apr 27, 2026 - 10:40:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0067 iter count: 1.0 +[Apr 27, 2026 - 10:40:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 10:40:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 10:43:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.807916 38.883366 38.728687 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.104492 133.175781 133.054688 +util 71.250000 79.000000 62.000000 +[Apr 27, 2026 - 10:43:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 30 : avg iteration time 179.27 seconds +[Apr 27, 2026 - 10:43:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 30 : data loading time 0.00 +[Apr 27, 2026 - 10:43:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 30 : avg forward pass time 80.66 +[Apr 27, 2026 - 10:43:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 30 : backward pass time 8.41 +[Apr 27, 2026 - 10:43:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 30 : optimizer step time 0.34 +[Apr 27, 2026 - 10:43:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 30-------------------- +[Apr 27, 2026 - 10:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 1387378.9977 iter count: 1.0 +[Apr 27, 2026 - 10:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.8293 iter count: 1.0 +[Apr 27, 2026 - 10:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 1387378.9977iter count: 1.0 +[Apr 27, 2026 - 10:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 10:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 3.8815iter count: 1.0 +[Apr 27, 2026 - 10:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.4186 iter count: 1.0 +[Apr 27, 2026 - 10:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.9235 iter count: 1.0 +[Apr 27, 2026 - 10:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 1781885.8750iter count: 1.0 +[Apr 27, 2026 - 10:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 17.0684iter count: 1.0 +[Apr 27, 2026 - 10:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 56341232.0000iter count: 1.0 +[Apr 27, 2026 - 10:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.8293iter count: 1.0 +[Apr 27, 2026 - 10:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.8383 iter count: 1.0 +[Apr 27, 2026 - 10:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 15940.6260iter count: 1.0 +[Apr 27, 2026 - 10:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.4044 iter count: 1.0 +[Apr 27, 2026 - 10:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 6692.6128iter count: 1.0 +[Apr 27, 2026 - 10:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 2.5962 iter count: 1.0 +[Apr 27, 2026 - 10:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 179.9101iter count: 1.0 +[Apr 27, 2026 - 10:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.9281 iter count: 1.0 +[Apr 27, 2026 - 10:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 19074.4277iter count: 1.0 +[Apr 27, 2026 - 10:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0069 iter count: 1.0 +[Apr 27, 2026 - 10:43:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 10:43:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 10:43:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808064 38.883377 38.728741 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.237793 67.437500 66.968750 +util 98.000000 99.000000 95.000000 +[Apr 27, 2026 - 10:43:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 31 : avg iteration time 49.35 seconds +[Apr 27, 2026 - 10:43:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 31 : data loading time 0.00 +[Apr 27, 2026 - 10:43:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 31 : avg forward pass time 12.13 +[Apr 27, 2026 - 10:43:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 31 : backward pass time 12.53 +[Apr 27, 2026 - 10:43:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 31 : optimizer step time 1.01 +[Apr 27, 2026 - 10:43:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 31-------------------- +[Apr 27, 2026 - 10:43:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0075 iter count: 1.0 +[Apr 27, 2026 - 10:43:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0075 iter count: 1.0 +[Apr 27, 2026 - 10:43:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 10:43:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 10:44:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808390 38.883404 38.728745 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.237793 67.476562 66.951172 +util 97.500000 99.000000 93.000000 +[Apr 27, 2026 - 10:44:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 32 : avg iteration time 49.19 seconds +[Apr 27, 2026 - 10:44:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 32 : data loading time 0.00 +[Apr 27, 2026 - 10:44:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 32 : avg forward pass time 12.55 +[Apr 27, 2026 - 10:44:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 32 : backward pass time 11.53 +[Apr 27, 2026 - 10:44:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 32 : optimizer step time 1.01 +[Apr 27, 2026 - 10:44:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 32-------------------- +[Apr 27, 2026 - 10:44:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0068 iter count: 1.0 +[Apr 27, 2026 - 10:44:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0068 iter count: 1.0 +[Apr 27, 2026 - 10:44:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 10:44:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 10:45:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808378 38.883404 38.728745 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.252441 67.476562 67.009766 +util 97.000000 99.000000 92.000000 +[Apr 27, 2026 - 10:45:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 33 : avg iteration time 47.38 seconds +[Apr 27, 2026 - 10:45:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 33 : data loading time 0.00 +[Apr 27, 2026 - 10:45:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 33 : avg forward pass time 11.08 +[Apr 27, 2026 - 10:45:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 33 : backward pass time 11.57 +[Apr 27, 2026 - 10:45:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 33 : optimizer step time 0.96 +[Apr 27, 2026 - 10:45:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 33-------------------- +[Apr 27, 2026 - 10:45:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0063 iter count: 1.0 +[Apr 27, 2026 - 10:45:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0063 iter count: 1.0 +[Apr 27, 2026 - 10:45:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 10:45:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 10:46:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808281 38.883404 38.728764 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.267090 67.476562 67.029297 +util 98.250000 99.000000 96.000000 +[Apr 27, 2026 - 10:46:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 34 : avg iteration time 53.57 seconds +[Apr 27, 2026 - 10:46:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 34 : data loading time 0.00 +[Apr 27, 2026 - 10:46:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 34 : avg forward pass time 14.10 +[Apr 27, 2026 - 10:46:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 34 : backward pass time 12.75 +[Apr 27, 2026 - 10:46:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 34 : optimizer step time 1.07 +[Apr 27, 2026 - 10:46:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 34-------------------- +[Apr 27, 2026 - 10:46:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0068 iter count: 1.0 +[Apr 27, 2026 - 10:46:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0068 iter count: 1.0 +[Apr 27, 2026 - 10:46:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 10:46:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 10:49:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808309 38.883476 38.728977 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.196777 133.232422 133.093750 +util 74.000000 78.000000 68.000000 +[Apr 27, 2026 - 10:49:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 35 : avg iteration time 172.69 seconds +[Apr 27, 2026 - 10:49:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 35 : data loading time 0.00 +[Apr 27, 2026 - 10:49:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 35 : avg forward pass time 77.81 +[Apr 27, 2026 - 10:49:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 35 : backward pass time 8.42 +[Apr 27, 2026 - 10:49:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 35 : optimizer step time 0.34 +[Apr 27, 2026 - 10:49:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 35-------------------- +[Apr 27, 2026 - 10:49:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 46855.4719 iter count: 1.0 +[Apr 27, 2026 - 10:49:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7386 iter count: 1.0 +[Apr 27, 2026 - 10:49:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 46855.4719iter count: 1.0 +[Apr 27, 2026 - 10:49:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 10:49:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 3.7613iter count: 1.0 +[Apr 27, 2026 - 10:49:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.9528 iter count: 1.0 +[Apr 27, 2026 - 10:49:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 6.8028 iter count: 1.0 +[Apr 27, 2026 - 10:49:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 58100.4531iter count: 1.0 +[Apr 27, 2026 - 10:49:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 6.7234 iter count: 1.0 +[Apr 27, 2026 - 10:49:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 810689.5000iter count: 1.0 +[Apr 27, 2026 - 10:49:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7386iter count: 1.0 +[Apr 27, 2026 - 10:49:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.9478 iter count: 1.0 +[Apr 27, 2026 - 10:49:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 2685.4131iter count: 1.0 +[Apr 27, 2026 - 10:49:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 0.9528 iter count: 1.0 +[Apr 27, 2026 - 10:49:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 6.7234 iter count: 1.0 +[Apr 27, 2026 - 10:49:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.2083 iter count: 1.0 +[Apr 27, 2026 - 10:49:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 33408.5430iter count: 1.0 +[Apr 27, 2026 - 10:49:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 2.6605 iter count: 1.0 +[Apr 27, 2026 - 10:49:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 204.5823iter count: 1.0 +[Apr 27, 2026 - 10:49:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0066 iter count: 1.0 +[Apr 27, 2026 - 10:49:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 10:49:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 10:50:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808443 38.883476 38.728977 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.247559 67.417969 66.968750 +util 98.000000 100.000000 93.000000 +[Apr 27, 2026 - 10:50:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 36 : avg iteration time 51.63 seconds +[Apr 27, 2026 - 10:50:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 36 : data loading time 0.00 +[Apr 27, 2026 - 10:50:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 36 : avg forward pass time 13.15 +[Apr 27, 2026 - 10:50:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 36 : backward pass time 11.55 +[Apr 27, 2026 - 10:50:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 36 : optimizer step time 1.32 +[Apr 27, 2026 - 10:50:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 36-------------------- +[Apr 27, 2026 - 10:50:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0086 iter count: 1.0 +[Apr 27, 2026 - 10:50:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[Apr 27, 2026 - 10:50:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 10:50:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 10:50:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808429 38.883465 38.728954 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.291504 67.417969 67.046875 +util 96.250000 99.000000 93.000000 +[Apr 27, 2026 - 10:50:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 37 : avg iteration time 47.45 seconds +[Apr 27, 2026 - 10:50:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 37 : data loading time 0.00 +[Apr 27, 2026 - 10:50:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 37 : avg forward pass time 11.12 +[Apr 27, 2026 - 10:50:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 37 : backward pass time 12.63 +[Apr 27, 2026 - 10:50:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 37 : optimizer step time 1.04 +[Apr 27, 2026 - 10:50:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 37-------------------- +[Apr 27, 2026 - 10:50:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0067 iter count: 1.0 +[Apr 27, 2026 - 10:50:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0067 iter count: 1.0 +[Apr 27, 2026 - 10:50:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 10:50:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 10:51:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808433 38.883480 38.728954 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.291504 67.378906 67.046875 +util 97.750000 99.000000 94.000000 +[Apr 27, 2026 - 10:51:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 38 : avg iteration time 51.69 seconds +[Apr 27, 2026 - 10:51:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 38 : data loading time 0.00 +[Apr 27, 2026 - 10:51:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 38 : avg forward pass time 13.79 +[Apr 27, 2026 - 10:51:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 38 : backward pass time 11.57 +[Apr 27, 2026 - 10:51:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 38 : optimizer step time 0.98 +[Apr 27, 2026 - 10:51:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 38-------------------- +[Apr 27, 2026 - 10:51:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0071 iter count: 1.0 +[Apr 27, 2026 - 10:51:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0071 iter count: 1.0 +[Apr 27, 2026 - 10:51:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 10:51:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 10:52:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808494 38.883480 38.728954 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 68.697754 70.269531 67.046875 +util 96.250000 100.000000 91.000000 +[Apr 27, 2026 - 10:52:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 39 : avg iteration time 50.29 seconds +[Apr 27, 2026 - 10:52:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 39 : data loading time 0.00 +[Apr 27, 2026 - 10:52:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 39 : avg forward pass time 12.48 +[Apr 27, 2026 - 10:52:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 39 : backward pass time 11.59 +[Apr 27, 2026 - 10:52:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 39 : optimizer step time 1.32 +[Apr 27, 2026 - 10:52:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 39-------------------- +[Apr 27, 2026 - 10:52:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0076 iter count: 1.0 +[Apr 27, 2026 - 10:52:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0076 iter count: 1.0 +[Apr 27, 2026 - 10:52:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 10:52:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 10:55:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808477 38.883518 38.728954 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.162598 133.251953 133.132812 +util 69.250000 75.000000 65.000000 +[Apr 27, 2026 - 10:55:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 40 : avg iteration time 171.43 seconds +[Apr 27, 2026 - 10:55:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 40 : data loading time 0.00 +[Apr 27, 2026 - 10:55:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 40 : avg forward pass time 77.13 +[Apr 27, 2026 - 10:55:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 40 : backward pass time 8.42 +[Apr 27, 2026 - 10:55:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 40 : optimizer step time 0.34 +[Apr 27, 2026 - 10:55:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 40-------------------- +[Apr 27, 2026 - 10:55:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 139973.3496 iter count: 1.0 +[Apr 27, 2026 - 10:55:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7710 iter count: 1.0 +[Apr 27, 2026 - 10:55:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 139973.3496iter count: 1.0 +[Apr 27, 2026 - 10:55:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 10:55:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.0018iter count: 1.0 +[Apr 27, 2026 - 10:55:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.7528 iter count: 1.0 +[Apr 27, 2026 - 10:55:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.4255 iter count: 1.0 +[Apr 27, 2026 - 10:55:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 175479.5625iter count: 1.0 +[Apr 27, 2026 - 10:55:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 4.5068 iter count: 1.0 +[Apr 27, 2026 - 10:55:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 2816314.7500iter count: 1.0 +[Apr 27, 2026 - 10:55:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7710iter count: 1.0 +[Apr 27, 2026 - 10:55:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.1666 iter count: 1.0 +[Apr 27, 2026 - 10:55:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 562.9415iter count: 1.0 +[Apr 27, 2026 - 10:55:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.4441 iter count: 1.0 +[Apr 27, 2026 - 10:55:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 980.5419iter count: 1.0 +[Apr 27, 2026 - 10:55:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 1.8331 iter count: 1.0 +[Apr 27, 2026 - 10:55:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 39.1017 iter count: 1.0 +[Apr 27, 2026 - 10:55:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.9479 iter count: 1.0 +[Apr 27, 2026 - 10:55:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 2685.9509iter count: 1.0 +[Apr 27, 2026 - 10:55:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0063 iter count: 1.0 +[Apr 27, 2026 - 10:55:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 10:55:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 10:56:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808473 38.883518 38.728954 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 66.969238 67.066406 66.910156 +util 97.500000 100.000000 93.000000 +[Apr 27, 2026 - 10:56:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 41 : avg iteration time 54.58 seconds +[Apr 27, 2026 - 10:56:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 41 : data loading time 0.00 +[Apr 27, 2026 - 10:56:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 41 : avg forward pass time 14.65 +[Apr 27, 2026 - 10:56:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 41 : backward pass time 11.61 +[Apr 27, 2026 - 10:56:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 41 : optimizer step time 1.36 +[Apr 27, 2026 - 10:56:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 41-------------------- +[Apr 27, 2026 - 10:56:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0076 iter count: 1.0 +[Apr 27, 2026 - 10:56:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0076 iter count: 1.0 +[Apr 27, 2026 - 10:56:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 10:56:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 10:57:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808481 38.883518 38.728958 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.286621 67.417969 66.970703 +util 97.000000 99.000000 94.000000 +[Apr 27, 2026 - 10:57:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 42 : avg iteration time 50.02 seconds +[Apr 27, 2026 - 10:57:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 42 : data loading time 0.00 +[Apr 27, 2026 - 10:57:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 42 : avg forward pass time 12.58 +[Apr 27, 2026 - 10:57:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 42 : backward pass time 12.31 +[Apr 27, 2026 - 10:57:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 42 : optimizer step time 0.99 +[Apr 27, 2026 - 10:57:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 42-------------------- +[Apr 27, 2026 - 10:57:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0075 iter count: 1.0 +[Apr 27, 2026 - 10:57:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0075 iter count: 1.0 +[Apr 27, 2026 - 10:57:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 10:57:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 10:58:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808477 38.883522 38.728962 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.096191 67.419922 66.968750 +util 98.250000 99.000000 96.000000 +[Apr 27, 2026 - 10:58:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 43 : avg iteration time 46.21 seconds +[Apr 27, 2026 - 10:58:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 43 : data loading time 0.00 +[Apr 27, 2026 - 10:58:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 43 : avg forward pass time 11.07 +[Apr 27, 2026 - 10:58:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 43 : backward pass time 11.56 +[Apr 27, 2026 - 10:58:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 43 : optimizer step time 0.97 +[Apr 27, 2026 - 10:58:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 43-------------------- +[Apr 27, 2026 - 10:58:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0064 iter count: 1.0 +[Apr 27, 2026 - 10:58:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0064 iter count: 1.0 +[Apr 27, 2026 - 10:58:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 10:58:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 10:58:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808610 38.883526 38.728973 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.066895 67.378906 66.951172 +util 94.250000 99.000000 91.000000 +[Apr 27, 2026 - 10:58:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 44 : avg iteration time 50.14 seconds +[Apr 27, 2026 - 10:58:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 44 : data loading time 0.00 +[Apr 27, 2026 - 10:58:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 44 : avg forward pass time 12.46 +[Apr 27, 2026 - 10:58:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 44 : backward pass time 11.54 +[Apr 27, 2026 - 10:58:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 44 : optimizer step time 1.01 +[Apr 27, 2026 - 10:58:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 44-------------------- +[Apr 27, 2026 - 10:58:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0076 iter count: 1.0 +[Apr 27, 2026 - 10:58:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0076 iter count: 1.0 +[Apr 27, 2026 - 10:58:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 10:58:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 11:01:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808323 38.883572 38.728970 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.187012 133.232422 133.093750 +util 71.750000 78.000000 63.000000 +[Apr 27, 2026 - 11:01:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 45 : avg iteration time 170.97 seconds +[Apr 27, 2026 - 11:01:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 45 : data loading time 0.00 +[Apr 27, 2026 - 11:01:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 45 : avg forward pass time 76.31 +[Apr 27, 2026 - 11:01:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 45 : backward pass time 8.41 +[Apr 27, 2026 - 11:01:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 45 : optimizer step time 0.34 +[Apr 27, 2026 - 11:01:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 45-------------------- +[Apr 27, 2026 - 11:01:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 99365.2983 iter count: 1.0 +[Apr 27, 2026 - 11:01:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7777 iter count: 1.0 +[Apr 27, 2026 - 11:01:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 99365.2983iter count: 1.0 +[Apr 27, 2026 - 11:01:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 11:01:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.5406iter count: 1.0 +[Apr 27, 2026 - 11:01:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.9408 iter count: 1.0 +[Apr 27, 2026 - 11:01:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.1741 iter count: 1.0 +[Apr 27, 2026 - 11:01:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 151634.8438iter count: 1.0 +[Apr 27, 2026 - 11:01:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 48.5038iter count: 1.0 +[Apr 27, 2026 - 11:01:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 1703481.5000iter count: 1.0 +[Apr 27, 2026 - 11:01:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7777iter count: 1.0 +[Apr 27, 2026 - 11:01:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 6.0881 iter count: 1.0 +[Apr 27, 2026 - 11:01:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 194132.1250iter count: 1.0 +[Apr 27, 2026 - 11:01:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 2.8084 iter count: 1.0 +[Apr 27, 2026 - 11:01:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 275.0227iter count: 1.0 +[Apr 27, 2026 - 11:01:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.3275 iter count: 1.0 +[Apr 27, 2026 - 11:01:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 42407.9180iter count: 1.0 +[Apr 27, 2026 - 11:01:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.9748 iter count: 1.0 +[Apr 27, 2026 - 11:01:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 154761.1719iter count: 1.0 +[Apr 27, 2026 - 11:01:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0070 iter count: 1.0 +[Apr 27, 2026 - 11:01:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 11:01:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 11:02:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808397 38.883575 38.728996 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.257324 67.457031 67.007812 +util 98.500000 100.000000 95.000000 +[Apr 27, 2026 - 11:02:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 46 : avg iteration time 54.68 seconds +[Apr 27, 2026 - 11:02:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 46 : data loading time 0.00 +[Apr 27, 2026 - 11:02:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 46 : avg forward pass time 14.65 +[Apr 27, 2026 - 11:02:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 46 : backward pass time 11.61 +[Apr 27, 2026 - 11:02:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 46 : optimizer step time 1.37 +[Apr 27, 2026 - 11:02:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 46-------------------- +[Apr 27, 2026 - 11:02:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0077 iter count: 1.0 +[Apr 27, 2026 - 11:02:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0077 iter count: 1.0 +[Apr 27, 2026 - 11:02:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 11:02:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 11:03:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808393 38.883579 38.728973 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.247559 67.437500 67.048828 +util 97.250000 100.000000 91.000000 +[Apr 27, 2026 - 11:03:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 47 : avg iteration time 49.92 seconds +[Apr 27, 2026 - 11:03:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 47 : data loading time 0.00 +[Apr 27, 2026 - 11:03:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 47 : avg forward pass time 12.62 +[Apr 27, 2026 - 11:03:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 47 : backward pass time 12.12 +[Apr 27, 2026 - 11:03:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 47 : optimizer step time 0.99 +[Apr 27, 2026 - 11:03:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 47-------------------- +[Apr 27, 2026 - 11:03:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0084 iter count: 1.0 +[Apr 27, 2026 - 11:03:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[Apr 27, 2026 - 11:03:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 11:03:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 11:04:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808393 38.883579 38.728985 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.208496 67.339844 67.066406 +util 96.500000 99.000000 91.000000 +[Apr 27, 2026 - 11:04:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 48 : avg iteration time 52.31 seconds +[Apr 27, 2026 - 11:04:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 48 : data loading time 0.00 +[Apr 27, 2026 - 11:04:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 48 : avg forward pass time 14.00 +[Apr 27, 2026 - 11:04:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 48 : backward pass time 11.60 +[Apr 27, 2026 - 11:04:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 48 : optimizer step time 0.97 +[Apr 27, 2026 - 11:04:21 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 48-------------------- +[Apr 27, 2026 - 11:04:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0083 iter count: 1.0 +[Apr 27, 2026 - 11:04:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[Apr 27, 2026 - 11:04:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 11:04:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 11:05:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808483 38.883587 38.729038 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.286621 67.457031 67.105469 +util 96.750000 100.000000 93.000000 +[Apr 27, 2026 - 11:05:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 49 : avg iteration time 53.52 seconds +[Apr 27, 2026 - 11:05:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 49 : data loading time 0.00 +[Apr 27, 2026 - 11:05:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 49 : avg forward pass time 14.05 +[Apr 27, 2026 - 11:05:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 49 : backward pass time 12.45 +[Apr 27, 2026 - 11:05:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 49 : optimizer step time 1.36 +[Apr 27, 2026 - 11:05:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 49-------------------- +[Apr 27, 2026 - 11:05:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0080 iter count: 1.0 +[Apr 27, 2026 - 11:05:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0080 iter count: 1.0 +[Apr 27, 2026 - 11:05:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 11:05:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 11:08:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808485 38.883663 38.728992 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.187012 133.250000 133.093750 +util 65.750000 73.000000 61.000000 +[Apr 27, 2026 - 11:08:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 50 : avg iteration time 168.81 seconds +[Apr 27, 2026 - 11:08:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 50 : data loading time 0.00 +[Apr 27, 2026 - 11:08:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 50 : avg forward pass time 75.67 +[Apr 27, 2026 - 11:08:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 50 : backward pass time 8.44 +[Apr 27, 2026 - 11:08:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 50 : optimizer step time 0.34 +[Apr 27, 2026 - 11:08:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 50-------------------- +[Apr 27, 2026 - 11:08:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 122894.0544 iter count: 1.0 +[Apr 27, 2026 - 11:08:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7793 iter count: 1.0 +[Apr 27, 2026 - 11:08:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 122894.0544iter count: 1.0 +[Apr 27, 2026 - 11:08:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 11:08:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.5753iter count: 1.0 +[Apr 27, 2026 - 11:08:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.8334 iter count: 1.0 +[Apr 27, 2026 - 11:08:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.4040 iter count: 1.0 +[Apr 27, 2026 - 11:08:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 175423.4062iter count: 1.0 +[Apr 27, 2026 - 11:08:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 39.1299iter count: 1.0 +[Apr 27, 2026 - 11:08:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 2697707.2500iter count: 1.0 +[Apr 27, 2026 - 11:08:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7793iter count: 1.0 +[Apr 27, 2026 - 11:08:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 6.8071 iter count: 1.0 +[Apr 27, 2026 - 11:08:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 817636.3125iter count: 1.0 +[Apr 27, 2026 - 11:08:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 2.0224 iter count: 1.0 +[Apr 27, 2026 - 11:08:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 57.0977 iter count: 1.0 +[Apr 27, 2026 - 11:08:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 1.8334 iter count: 1.0 +[Apr 27, 2026 - 11:08:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 39.1299 iter count: 1.0 +[Apr 27, 2026 - 11:08:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.2262 iter count: 1.0 +[Apr 27, 2026 - 11:08:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 634.1937iter count: 1.0 +[Apr 27, 2026 - 11:08:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0073 iter count: 1.0 +[Apr 27, 2026 - 11:08:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 11:08:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 11:08:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808554 38.883690 38.728992 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.848145 70.269531 67.007812 +util 97.250000 100.000000 93.000000 +[Apr 27, 2026 - 11:08:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 51 : avg iteration time 51.42 seconds +[Apr 27, 2026 - 11:08:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 51 : data loading time 0.00 +[Apr 27, 2026 - 11:08:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 51 : avg forward pass time 13.11 +[Apr 27, 2026 - 11:08:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 51 : backward pass time 11.52 +[Apr 27, 2026 - 11:08:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 51 : optimizer step time 1.03 +[Apr 27, 2026 - 11:08:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 51-------------------- +[Apr 27, 2026 - 11:08:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0078 iter count: 1.0 +[Apr 27, 2026 - 11:08:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0078 iter count: 1.0 +[Apr 27, 2026 - 11:08:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 11:08:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 11:09:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808571 38.883667 38.729015 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.081543 67.320312 66.988281 +util 98.000000 99.000000 96.000000 +[Apr 27, 2026 - 11:09:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 52 : avg iteration time 49.83 seconds +[Apr 27, 2026 - 11:09:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 52 : data loading time 0.00 +[Apr 27, 2026 - 11:09:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 52 : avg forward pass time 12.59 +[Apr 27, 2026 - 11:09:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 52 : backward pass time 12.08 +[Apr 27, 2026 - 11:09:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 52 : optimizer step time 0.99 +[Apr 27, 2026 - 11:09:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 52-------------------- +[Apr 27, 2026 - 11:09:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0089 iter count: 1.0 +[Apr 27, 2026 - 11:09:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[Apr 27, 2026 - 11:09:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 11:09:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 11:10:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808570 38.883690 38.729015 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.130371 67.378906 67.046875 +util 96.500000 99.000000 91.000000 +[Apr 27, 2026 - 11:10:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 53 : avg iteration time 49.62 seconds +[Apr 27, 2026 - 11:10:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 53 : data loading time 0.00 +[Apr 27, 2026 - 11:10:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 53 : avg forward pass time 12.66 +[Apr 27, 2026 - 11:10:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 53 : backward pass time 11.52 +[Apr 27, 2026 - 11:10:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 53 : optimizer step time 0.96 +[Apr 27, 2026 - 11:10:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 53-------------------- +[Apr 27, 2026 - 11:10:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0074 iter count: 1.0 +[Apr 27, 2026 - 11:10:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0074 iter count: 1.0 +[Apr 27, 2026 - 11:10:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 11:10:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 11:11:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808569 38.883667 38.729031 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.062012 67.222656 66.968750 +util 98.500000 100.000000 96.000000 +[Apr 27, 2026 - 11:11:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 54 : avg iteration time 49.86 seconds +[Apr 27, 2026 - 11:11:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 54 : data loading time 0.00 +[Apr 27, 2026 - 11:11:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 54 : avg forward pass time 12.55 +[Apr 27, 2026 - 11:11:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 54 : backward pass time 11.52 +[Apr 27, 2026 - 11:11:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 54 : optimizer step time 1.18 +[Apr 27, 2026 - 11:11:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 54-------------------- +[Apr 27, 2026 - 11:11:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0078 iter count: 1.0 +[Apr 27, 2026 - 11:11:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0078 iter count: 1.0 +[Apr 27, 2026 - 11:11:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 11:11:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 11:14:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808345 38.883694 38.729313 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.138184 133.210938 133.093750 +util 66.250000 74.000000 60.000000 +[Apr 27, 2026 - 11:14:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 55 : avg iteration time 175.15 seconds +[Apr 27, 2026 - 11:14:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 55 : data loading time 0.00 +[Apr 27, 2026 - 11:14:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 55 : avg forward pass time 77.61 +[Apr 27, 2026 - 11:14:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 55 : backward pass time 9.53 +[Apr 27, 2026 - 11:14:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 55 : optimizer step time 0.44 +[Apr 27, 2026 - 11:14:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 55-------------------- +[Apr 27, 2026 - 11:14:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 65408.7656 iter count: 1.0 +[Apr 27, 2026 - 11:14:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7870 iter count: 1.0 +[Apr 27, 2026 - 11:14:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 65408.7656iter count: 1.0 +[Apr 27, 2026 - 11:14:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 11:14:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 3.7972iter count: 1.0 +[Apr 27, 2026 - 11:14:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.9552 iter count: 1.0 +[Apr 27, 2026 - 11:14:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.0010 iter count: 1.0 +[Apr 27, 2026 - 11:14:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 84107.2500iter count: 1.0 +[Apr 27, 2026 - 11:14:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 6.7558 iter count: 1.0 +[Apr 27, 2026 - 11:14:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 1205110.6250iter count: 1.0 +[Apr 27, 2026 - 11:14:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7870iter count: 1.0 +[Apr 27, 2026 - 11:14:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 7.0010 iter count: 1.0 +[Apr 27, 2026 - 11:14:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 1205110.6250iter count: 1.0 +[Apr 27, 2026 - 11:14:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 2.1394 iter count: 1.0 +[Apr 27, 2026 - 11:14:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 72.1543 iter count: 1.0 +[Apr 27, 2026 - 11:14:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.0602 iter count: 1.0 +[Apr 27, 2026 - 11:14:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 3362.3059iter count: 1.0 +[Apr 27, 2026 - 11:14:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.2492 iter count: 1.0 +[Apr 27, 2026 - 11:14:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 664.0800iter count: 1.0 +[Apr 27, 2026 - 11:14:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0080 iter count: 1.0 +[Apr 27, 2026 - 11:14:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 11:14:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 11:15:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808347 38.883678 38.729336 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.198730 67.419922 67.007812 +util 98.500000 99.000000 97.000000 +[Apr 27, 2026 - 11:15:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 56 : avg iteration time 49.06 seconds +[Apr 27, 2026 - 11:15:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 56 : data loading time 0.00 +[Apr 27, 2026 - 11:15:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 56 : avg forward pass time 12.15 +[Apr 27, 2026 - 11:15:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 56 : backward pass time 12.21 +[Apr 27, 2026 - 11:15:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 56 : optimizer step time 0.98 +[Apr 27, 2026 - 11:15:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 56-------------------- +[Apr 27, 2026 - 11:15:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0075 iter count: 1.0 +[Apr 27, 2026 - 11:15:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0075 iter count: 1.0 +[Apr 27, 2026 - 11:15:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 11:15:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 11:16:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808729 38.883797 38.729298 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.374512 67.458984 67.164062 +util 97.000000 99.000000 92.000000 +[Apr 27, 2026 - 11:16:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 57 : avg iteration time 52.15 seconds +[Apr 27, 2026 - 11:16:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 57 : data loading time 0.00 +[Apr 27, 2026 - 11:16:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 57 : avg forward pass time 13.98 +[Apr 27, 2026 - 11:16:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 57 : backward pass time 11.53 +[Apr 27, 2026 - 11:16:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 57 : optimizer step time 0.97 +[Apr 27, 2026 - 11:16:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 57-------------------- +[Apr 27, 2026 - 11:16:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0086 iter count: 1.0 +[Apr 27, 2026 - 11:16:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[Apr 27, 2026 - 11:16:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 11:16:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 11:16:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808853 38.883820 38.729309 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.188965 67.439453 66.988281 +util 97.750000 100.000000 93.000000 +[Apr 27, 2026 - 11:16:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 58 : avg iteration time 50.41 seconds +[Apr 27, 2026 - 11:16:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 58 : data loading time 0.00 +[Apr 27, 2026 - 11:16:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 58 : avg forward pass time 12.79 +[Apr 27, 2026 - 11:16:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 58 : backward pass time 11.56 +[Apr 27, 2026 - 11:16:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 58 : optimizer step time 1.29 +[Apr 27, 2026 - 11:16:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 58-------------------- +[Apr 27, 2026 - 11:16:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0072 iter count: 1.0 +[Apr 27, 2026 - 11:16:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0072 iter count: 1.0 +[Apr 27, 2026 - 11:16:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 11:16:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 11:17:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808851 38.883823 38.729298 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 68.111816 70.230469 67.380859 +util 98.500000 99.000000 97.000000 +[Apr 27, 2026 - 11:17:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 59 : avg iteration time 50.64 seconds +[Apr 27, 2026 - 11:17:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 59 : data loading time 0.00 +[Apr 27, 2026 - 11:17:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 59 : avg forward pass time 12.53 +[Apr 27, 2026 - 11:17:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 59 : backward pass time 12.92 +[Apr 27, 2026 - 11:17:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 59 : optimizer step time 1.10 +[Apr 27, 2026 - 11:17:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 59-------------------- +[Apr 27, 2026 - 11:17:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0072 iter count: 1.0 +[Apr 27, 2026 - 11:17:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0072 iter count: 1.0 +[Apr 27, 2026 - 11:17:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 11:17:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 11:20:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808329 38.883831 38.729298 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.191895 133.230469 133.152344 +util 72.750000 76.000000 67.000000 +[Apr 27, 2026 - 11:20:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 60 : avg iteration time 172.80 seconds +[Apr 27, 2026 - 11:20:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 60 : data loading time 0.00 +[Apr 27, 2026 - 11:20:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 60 : avg forward pass time 77.50 +[Apr 27, 2026 - 11:20:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 60 : backward pass time 8.37 +[Apr 27, 2026 - 11:20:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 60 : optimizer step time 0.34 +[Apr 27, 2026 - 11:20:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 60-------------------- +[Apr 27, 2026 - 11:20:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 10813.2844 iter count: 1.0 +[Apr 27, 2026 - 11:20:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7550 iter count: 1.0 +[Apr 27, 2026 - 11:20:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 10813.2844iter count: 1.0 +[Apr 27, 2026 - 11:20:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 11:20:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 3.6967iter count: 1.0 +[Apr 27, 2026 - 11:20:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.2151 iter count: 1.0 +[Apr 27, 2026 - 11:20:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 5.8246 iter count: 1.0 +[Apr 27, 2026 - 11:20:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 14389.3047iter count: 1.0 +[Apr 27, 2026 - 11:20:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 11.3604iter count: 1.0 +[Apr 27, 2026 - 11:20:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 114607.5703iter count: 1.0 +[Apr 27, 2026 - 11:20:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7550iter count: 1.0 +[Apr 27, 2026 - 11:20:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 2.8849 iter count: 1.0 +[Apr 27, 2026 - 11:20:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 320.4951iter count: 1.0 +[Apr 27, 2026 - 11:20:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.0614 iter count: 1.0 +[Apr 27, 2026 - 11:20:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 24903.5957iter count: 1.0 +[Apr 27, 2026 - 11:20:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 2.6041 iter count: 1.0 +[Apr 27, 2026 - 11:20:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 182.7760iter count: 1.0 +[Apr 27, 2026 - 11:20:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.5089 iter count: 1.0 +[Apr 27, 2026 - 11:20:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 8249.2188iter count: 1.0 +[Apr 27, 2026 - 11:20:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0076 iter count: 1.0 +[Apr 27, 2026 - 11:20:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 11:20:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 11:21:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808398 38.883854 38.729298 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.232910 67.478516 67.027344 +util 95.500000 99.000000 91.000000 +[Apr 27, 2026 - 11:21:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 61 : avg iteration time 54.64 seconds +[Apr 27, 2026 - 11:21:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 61 : data loading time 0.00 +[Apr 27, 2026 - 11:21:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 61 : avg forward pass time 14.81 +[Apr 27, 2026 - 11:21:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 61 : backward pass time 11.56 +[Apr 27, 2026 - 11:21:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 61 : optimizer step time 0.98 +[Apr 27, 2026 - 11:21:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 61-------------------- +[Apr 27, 2026 - 11:21:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0103 iter count: 1.0 +[Apr 27, 2026 - 11:21:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[Apr 27, 2026 - 11:21:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 11:21:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 11:22:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808761 38.883831 38.729298 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.154785 67.437500 67.046875 +util 98.500000 99.000000 97.000000 +[Apr 27, 2026 - 11:22:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 62 : avg iteration time 50.39 seconds +[Apr 27, 2026 - 11:22:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 62 : data loading time 0.00 +[Apr 27, 2026 - 11:22:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 62 : avg forward pass time 12.57 +[Apr 27, 2026 - 11:22:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 62 : backward pass time 11.54 +[Apr 27, 2026 - 11:22:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 62 : optimizer step time 1.07 +[Apr 27, 2026 - 11:22:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 62-------------------- +[Apr 27, 2026 - 11:22:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[Apr 27, 2026 - 11:22:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[Apr 27, 2026 - 11:22:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 11:22:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 11:23:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808845 38.883831 38.729298 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.022949 67.046875 66.968750 +util 97.500000 99.000000 93.000000 +[Apr 27, 2026 - 11:23:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 63 : avg iteration time 47.93 seconds +[Apr 27, 2026 - 11:23:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 63 : data loading time 0.00 +[Apr 27, 2026 - 11:23:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 63 : avg forward pass time 11.04 +[Apr 27, 2026 - 11:23:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 63 : backward pass time 13.07 +[Apr 27, 2026 - 11:23:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 63 : optimizer step time 1.20 +[Apr 27, 2026 - 11:23:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 63-------------------- +[Apr 27, 2026 - 11:23:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0075 iter count: 1.0 +[Apr 27, 2026 - 11:23:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0075 iter count: 1.0 +[Apr 27, 2026 - 11:23:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 11:23:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 11:23:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808841 38.883842 38.729298 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.140137 67.417969 67.009766 +util 98.250000 99.000000 96.000000 +[Apr 27, 2026 - 11:23:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 64 : avg iteration time 49.97 seconds +[Apr 27, 2026 - 11:23:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 64 : data loading time 0.00 +[Apr 27, 2026 - 11:23:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 64 : avg forward pass time 12.91 +[Apr 27, 2026 - 11:23:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 64 : backward pass time 11.61 +[Apr 27, 2026 - 11:23:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 64 : optimizer step time 0.97 +[Apr 27, 2026 - 11:23:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 64-------------------- +[Apr 27, 2026 - 11:23:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[Apr 27, 2026 - 11:23:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[Apr 27, 2026 - 11:23:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 11:23:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 11:26:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808640 38.883862 38.729301 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.123535 133.154297 133.113281 +util 71.250000 75.000000 65.000000 +[Apr 27, 2026 - 11:26:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 65 : avg iteration time 177.46 seconds +[Apr 27, 2026 - 11:26:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 65 : data loading time 0.00 +[Apr 27, 2026 - 11:26:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 65 : avg forward pass time 80.03 +[Apr 27, 2026 - 11:26:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 65 : backward pass time 8.40 +[Apr 27, 2026 - 11:26:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 65 : optimizer step time 0.34 +[Apr 27, 2026 - 11:26:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 65-------------------- +[Apr 27, 2026 - 11:26:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 199608.7803 iter count: 1.0 +[Apr 27, 2026 - 11:26:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7221 iter count: 1.0 +[Apr 27, 2026 - 11:26:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 199608.7803iter count: 1.0 +[Apr 27, 2026 - 11:26:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 11:26:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.3575iter count: 1.0 +[Apr 27, 2026 - 11:26:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.3916 iter count: 1.0 +[Apr 27, 2026 - 11:26:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.5855 iter count: 1.0 +[Apr 27, 2026 - 11:26:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 281807.0312iter count: 1.0 +[Apr 27, 2026 - 11:26:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 16.1715iter count: 1.0 +[Apr 27, 2026 - 11:26:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 3878930.7500iter count: 1.0 +[Apr 27, 2026 - 11:26:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7221iter count: 1.0 +[Apr 27, 2026 - 11:26:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.4170 iter count: 1.0 +[Apr 27, 2026 - 11:26:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 928.8932iter count: 1.0 +[Apr 27, 2026 - 11:26:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.9733 iter count: 1.0 +[Apr 27, 2026 - 11:26:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 2826.0149iter count: 1.0 +[Apr 27, 2026 - 11:26:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.7128 iter count: 1.0 +[Apr 27, 2026 - 11:26:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 1678.5153iter count: 1.0 +[Apr 27, 2026 - 11:26:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.5543 iter count: 1.0 +[Apr 27, 2026 - 11:26:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 1222.5359iter count: 1.0 +[Apr 27, 2026 - 11:26:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[Apr 27, 2026 - 11:26:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 11:26:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 11:27:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808699 38.883862 38.729301 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.306152 67.458984 66.929688 +util 98.750000 100.000000 97.000000 +[Apr 27, 2026 - 11:27:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 66 : avg iteration time 51.82 seconds +[Apr 27, 2026 - 11:27:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 66 : data loading time 0.00 +[Apr 27, 2026 - 11:27:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 66 : avg forward pass time 13.24 +[Apr 27, 2026 - 11:27:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 66 : backward pass time 11.55 +[Apr 27, 2026 - 11:27:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 66 : optimizer step time 0.98 +[Apr 27, 2026 - 11:27:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 66-------------------- +[Apr 27, 2026 - 11:27:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[Apr 27, 2026 - 11:27:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[Apr 27, 2026 - 11:27:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 11:27:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 11:28:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808836 38.883869 38.729309 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.330566 67.457031 67.007812 +util 97.250000 100.000000 91.000000 +[Apr 27, 2026 - 11:28:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 67 : avg iteration time 51.22 seconds +[Apr 27, 2026 - 11:28:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 67 : data loading time 0.00 +[Apr 27, 2026 - 11:28:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 67 : avg forward pass time 12.85 +[Apr 27, 2026 - 11:28:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 67 : backward pass time 12.34 +[Apr 27, 2026 - 11:28:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 67 : optimizer step time 1.58 +[Apr 27, 2026 - 11:28:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 67-------------------- +[Apr 27, 2026 - 11:28:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[Apr 27, 2026 - 11:28:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[Apr 27, 2026 - 11:28:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 11:28:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 11:29:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808829 38.883877 38.729298 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.242676 67.515625 66.949219 +util 97.250000 99.000000 92.000000 +[Apr 27, 2026 - 11:29:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 68 : avg iteration time 52.73 seconds +[Apr 27, 2026 - 11:29:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 68 : data loading time 0.00 +[Apr 27, 2026 - 11:29:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 68 : avg forward pass time 14.30 +[Apr 27, 2026 - 11:29:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 68 : backward pass time 11.59 +[Apr 27, 2026 - 11:29:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 68 : optimizer step time 0.98 +[Apr 27, 2026 - 11:29:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 68-------------------- +[Apr 27, 2026 - 11:29:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0086 iter count: 1.0 +[Apr 27, 2026 - 11:29:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[Apr 27, 2026 - 11:29:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 11:29:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 11:30:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808889 38.883865 38.729298 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.296387 67.574219 67.007812 +util 98.250000 100.000000 95.000000 +[Apr 27, 2026 - 11:30:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 69 : avg iteration time 52.83 seconds +[Apr 27, 2026 - 11:30:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 69 : data loading time 0.00 +[Apr 27, 2026 - 11:30:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 69 : avg forward pass time 14.05 +[Apr 27, 2026 - 11:30:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 69 : backward pass time 11.56 +[Apr 27, 2026 - 11:30:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 69 : optimizer step time 0.98 +[Apr 27, 2026 - 11:30:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 69-------------------- +[Apr 27, 2026 - 11:30:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0084 iter count: 1.0 +[Apr 27, 2026 - 11:30:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[Apr 27, 2026 - 11:30:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 11:30:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 11:33:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808738 38.883987 38.729321 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.158203 133.171875 133.136719 +util 72.250000 76.000000 66.000000 +[Apr 27, 2026 - 11:33:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 70 : avg iteration time 174.61 seconds +[Apr 27, 2026 - 11:33:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 70 : data loading time 0.00 +[Apr 27, 2026 - 11:33:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 70 : avg forward pass time 78.83 +[Apr 27, 2026 - 11:33:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 70 : backward pass time 8.36 +[Apr 27, 2026 - 11:33:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 70 : optimizer step time 0.33 +[Apr 27, 2026 - 11:33:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 70-------------------- +[Apr 27, 2026 - 11:33:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 151555.6016 iter count: 1.0 +[Apr 27, 2026 - 11:33:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7200 iter count: 1.0 +[Apr 27, 2026 - 11:33:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 151555.6016iter count: 1.0 +[Apr 27, 2026 - 11:33:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 11:33:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.2003iter count: 1.0 +[Apr 27, 2026 - 11:33:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.7672 iter count: 1.0 +[Apr 27, 2026 - 11:33:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.7863 iter count: 1.0 +[Apr 27, 2026 - 11:33:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 243720.4375iter count: 1.0 +[Apr 27, 2026 - 11:33:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 4.6384 iter count: 1.0 +[Apr 27, 2026 - 11:33:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 5795038.0000iter count: 1.0 +[Apr 27, 2026 - 11:33:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7200iter count: 1.0 +[Apr 27, 2026 - 11:33:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 2.2681 iter count: 1.0 +[Apr 27, 2026 - 11:33:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 93.3354 iter count: 1.0 +[Apr 27, 2026 - 11:33:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.2550 iter count: 1.0 +[Apr 27, 2026 - 11:33:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 4963.9180iter count: 1.0 +[Apr 27, 2026 - 11:33:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 1.6458 iter count: 1.0 +[Apr 27, 2026 - 11:33:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 26.8844 iter count: 1.0 +[Apr 27, 2026 - 11:33:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.9444 iter count: 1.0 +[Apr 27, 2026 - 11:33:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 2667.0574iter count: 1.0 +[Apr 27, 2026 - 11:33:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0076 iter count: 1.0 +[Apr 27, 2026 - 11:33:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 11:33:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 11:34:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808735 38.883987 38.729298 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.140137 67.417969 66.970703 +util 97.000000 100.000000 92.000000 +[Apr 27, 2026 - 11:34:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 71 : avg iteration time 52.82 seconds +[Apr 27, 2026 - 11:34:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 71 : data loading time 0.00 +[Apr 27, 2026 - 11:34:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 71 : avg forward pass time 13.60 +[Apr 27, 2026 - 11:34:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 71 : backward pass time 12.53 +[Apr 27, 2026 - 11:34:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 71 : optimizer step time 1.55 +[Apr 27, 2026 - 11:34:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 71-------------------- +[Apr 27, 2026 - 11:34:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[Apr 27, 2026 - 11:34:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[Apr 27, 2026 - 11:34:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 11:34:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 11:35:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808743 38.884010 38.729321 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.105957 67.476562 66.970703 +util 98.750000 100.000000 96.000000 +[Apr 27, 2026 - 11:35:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 72 : avg iteration time 50.39 seconds +[Apr 27, 2026 - 11:35:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 72 : data loading time 0.00 +[Apr 27, 2026 - 11:35:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 72 : avg forward pass time 12.61 +[Apr 27, 2026 - 11:35:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 72 : backward pass time 12.55 +[Apr 27, 2026 - 11:35:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 72 : optimizer step time 1.02 +[Apr 27, 2026 - 11:35:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 72-------------------- +[Apr 27, 2026 - 11:35:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[Apr 27, 2026 - 11:35:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[Apr 27, 2026 - 11:35:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 11:35:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 11:35:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808738 38.883987 38.729321 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.101074 67.437500 66.951172 +util 98.500000 99.000000 97.000000 +[Apr 27, 2026 - 11:35:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 73 : avg iteration time 51.74 seconds +[Apr 27, 2026 - 11:35:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 73 : data loading time 0.00 +[Apr 27, 2026 - 11:35:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 73 : avg forward pass time 13.81 +[Apr 27, 2026 - 11:35:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 73 : backward pass time 11.53 +[Apr 27, 2026 - 11:35:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 73 : optimizer step time 0.98 +[Apr 27, 2026 - 11:35:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 73-------------------- +[Apr 27, 2026 - 11:35:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0084 iter count: 1.0 +[Apr 27, 2026 - 11:35:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[Apr 27, 2026 - 11:35:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 11:35:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 11:36:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.808742 38.883987 38.729313 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.188965 67.419922 66.968750 +util 98.000000 99.000000 95.000000 +[Apr 27, 2026 - 11:36:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 74 : avg iteration time 53.78 seconds +[Apr 27, 2026 - 11:36:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 74 : data loading time 0.00 +[Apr 27, 2026 - 11:36:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 74 : avg forward pass time 14.20 +[Apr 27, 2026 - 11:36:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 74 : backward pass time 12.08 +[Apr 27, 2026 - 11:36:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 74 : optimizer step time 1.56 +[Apr 27, 2026 - 11:36:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 74-------------------- +[Apr 27, 2026 - 11:36:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[Apr 27, 2026 - 11:36:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[Apr 27, 2026 - 11:36:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 11:36:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 11:39:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.809327 38.884205 38.730347 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.211914 133.250000 133.132812 +util 72.250000 76.000000 67.000000 +[Apr 27, 2026 - 11:39:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 75 : avg iteration time 172.68 seconds +[Apr 27, 2026 - 11:39:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 75 : data loading time 0.00 +[Apr 27, 2026 - 11:39:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 75 : avg forward pass time 77.28 +[Apr 27, 2026 - 11:39:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 75 : backward pass time 8.34 +[Apr 27, 2026 - 11:39:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 75 : optimizer step time 0.35 +[Apr 27, 2026 - 11:39:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 75-------------------- +[Apr 27, 2026 - 11:39:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 51274.0337 iter count: 1.0 +[Apr 27, 2026 - 11:39:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6659 iter count: 1.0 +[Apr 27, 2026 - 11:39:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 51274.0337iter count: 1.0 +[Apr 27, 2026 - 11:39:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 11:39:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.3173iter count: 1.0 +[Apr 27, 2026 - 11:39:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.6720 iter count: 1.0 +[Apr 27, 2026 - 11:39:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 6.8522 iter count: 1.0 +[Apr 27, 2026 - 11:39:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 78949.3516iter count: 1.0 +[Apr 27, 2026 - 11:39:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 3.8341 iter count: 1.0 +[Apr 27, 2026 - 11:39:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 894844.1250iter count: 1.0 +[Apr 27, 2026 - 11:39:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6659iter count: 1.0 +[Apr 27, 2026 - 11:39:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.4309 iter count: 1.0 +[Apr 27, 2026 - 11:39:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 7057.0542iter count: 1.0 +[Apr 27, 2026 - 11:39:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.1892 iter count: 1.0 +[Apr 27, 2026 - 11:39:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 588.9424iter count: 1.0 +[Apr 27, 2026 - 11:39:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 1.9091 iter count: 1.0 +[Apr 27, 2026 - 11:39:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 45.5251 iter count: 1.0 +[Apr 27, 2026 - 11:39:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.4837 iter count: 1.0 +[Apr 27, 2026 - 11:39:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 7842.8433iter count: 1.0 +[Apr 27, 2026 - 11:39:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[Apr 27, 2026 - 11:39:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 11:39:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 11:40:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.809340 38.884228 38.730347 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.203613 67.417969 66.949219 +util 97.500000 99.000000 95.000000 +[Apr 27, 2026 - 11:40:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 76 : avg iteration time 54.91 seconds +[Apr 27, 2026 - 11:40:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 76 : data loading time 0.00 +[Apr 27, 2026 - 11:40:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 76 : avg forward pass time 14.77 +[Apr 27, 2026 - 11:40:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 76 : backward pass time 11.54 +[Apr 27, 2026 - 11:40:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 76 : optimizer step time 0.99 +[Apr 27, 2026 - 11:40:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 76-------------------- +[Apr 27, 2026 - 11:40:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0098 iter count: 1.0 +[Apr 27, 2026 - 11:40:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[Apr 27, 2026 - 11:40:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 11:40:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 11:41:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.809317 38.884205 38.730328 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.213379 67.417969 67.007812 +util 98.500000 100.000000 94.000000 +[Apr 27, 2026 - 11:41:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 77 : avg iteration time 54.25 seconds +[Apr 27, 2026 - 11:41:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 77 : data loading time 0.00 +[Apr 27, 2026 - 11:41:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 77 : avg forward pass time 14.21 +[Apr 27, 2026 - 11:41:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 77 : backward pass time 12.69 +[Apr 27, 2026 - 11:41:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 77 : optimizer step time 1.61 +[Apr 27, 2026 - 11:41:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 77-------------------- +[Apr 27, 2026 - 11:41:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0106 iter count: 1.0 +[Apr 27, 2026 - 11:41:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[Apr 27, 2026 - 11:41:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 11:41:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 11:42:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.809329 38.884209 38.730350 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 66.983887 67.027344 66.968750 +util 96.500000 99.000000 91.000000 +[Apr 27, 2026 - 11:42:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 78 : avg iteration time 52.78 seconds +[Apr 27, 2026 - 11:42:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 78 : data loading time 0.00 +[Apr 27, 2026 - 11:42:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 78 : avg forward pass time 14.32 +[Apr 27, 2026 - 11:42:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 78 : backward pass time 11.59 +[Apr 27, 2026 - 11:42:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 78 : optimizer step time 0.99 +[Apr 27, 2026 - 11:42:21 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 78-------------------- +[Apr 27, 2026 - 11:42:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0095 iter count: 1.0 +[Apr 27, 2026 - 11:42:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[Apr 27, 2026 - 11:42:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 11:42:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 11:43:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.809405 38.884209 38.730350 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.242676 67.457031 67.027344 +util 94.500000 99.000000 91.000000 +[Apr 27, 2026 - 11:43:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 79 : avg iteration time 52.92 seconds +[Apr 27, 2026 - 11:43:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 79 : data loading time 0.00 +[Apr 27, 2026 - 11:43:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 79 : avg forward pass time 14.04 +[Apr 27, 2026 - 11:43:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 79 : backward pass time 11.57 +[Apr 27, 2026 - 11:43:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 79 : optimizer step time 0.98 +[Apr 27, 2026 - 11:43:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 79-------------------- +[Apr 27, 2026 - 11:43:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0109 iter count: 1.0 +[Apr 27, 2026 - 11:43:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[Apr 27, 2026 - 11:43:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 11:43:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 11:46:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.809581 38.884861 38.730404 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.173828 133.173828 133.173828 +util 67.750000 74.000000 64.000000 +[Apr 27, 2026 - 11:46:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 80 : avg iteration time 169.82 seconds +[Apr 27, 2026 - 11:46:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 80 : data loading time 0.00 +[Apr 27, 2026 - 11:46:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 80 : avg forward pass time 76.39 +[Apr 27, 2026 - 11:46:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 80 : backward pass time 8.40 +[Apr 27, 2026 - 11:46:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 80 : optimizer step time 0.34 +[Apr 27, 2026 - 11:46:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 80-------------------- +[Apr 27, 2026 - 11:46:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 259295.2432 iter count: 1.0 +[Apr 27, 2026 - 11:46:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7326 iter count: 1.0 +[Apr 27, 2026 - 11:46:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 259295.2432iter count: 1.0 +[Apr 27, 2026 - 11:46:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 11:46:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 3.8158iter count: 1.0 +[Apr 27, 2026 - 11:46:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.3775 iter count: 1.0 +[Apr 27, 2026 - 11:46:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.0852 iter count: 1.0 +[Apr 27, 2026 - 11:46:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 408281.8125iter count: 1.0 +[Apr 27, 2026 - 11:46:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 2.1275 iter count: 1.0 +[Apr 27, 2026 - 11:46:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 10536327.0000iter count: 1.0 +[Apr 27, 2026 - 11:46:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7326iter count: 1.0 +[Apr 27, 2026 - 11:46:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 0.4785 iter count: 1.0 +[Apr 27, 2026 - 11:46:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 2.6038 iter count: 1.0 +[Apr 27, 2026 - 11:46:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 6.7889 iter count: 1.0 +[Apr 27, 2026 - 11:46:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 788404.3750iter count: 1.0 +[Apr 27, 2026 - 11:46:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.9374 iter count: 1.0 +[Apr 27, 2026 - 11:46:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 143603.0625iter count: 1.0 +[Apr 27, 2026 - 11:46:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 2.7089 iter count: 1.0 +[Apr 27, 2026 - 11:46:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 225.3726iter count: 1.0 +[Apr 27, 2026 - 11:46:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[Apr 27, 2026 - 11:46:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 11:46:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 11:46:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.809641 38.884838 38.730404 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.091309 67.380859 66.968750 +util 95.500000 99.000000 91.000000 +[Apr 27, 2026 - 11:46:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 81 : avg iteration time 49.16 seconds +[Apr 27, 2026 - 11:46:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 81 : data loading time 0.00 +[Apr 27, 2026 - 11:46:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 81 : avg forward pass time 11.83 +[Apr 27, 2026 - 11:46:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 81 : backward pass time 11.55 +[Apr 27, 2026 - 11:46:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 81 : optimizer step time 0.97 +[Apr 27, 2026 - 11:46:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 81-------------------- +[Apr 27, 2026 - 11:46:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0091 iter count: 1.0 +[Apr 27, 2026 - 11:46:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[Apr 27, 2026 - 11:46:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 11:46:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 11:47:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.810204 38.884838 38.730404 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.096191 67.380859 66.988281 +util 97.000000 100.000000 94.000000 +[Apr 27, 2026 - 11:47:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 82 : avg iteration time 57.86 seconds +[Apr 27, 2026 - 11:47:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 82 : data loading time 0.00 +[Apr 27, 2026 - 11:47:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 82 : avg forward pass time 12.54 +[Apr 27, 2026 - 11:47:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 82 : backward pass time 18.25 +[Apr 27, 2026 - 11:47:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 82 : optimizer step time 2.51 +[Apr 27, 2026 - 11:47:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 82-------------------- +[Apr 27, 2026 - 11:47:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0094 iter count: 1.0 +[Apr 27, 2026 - 11:47:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[Apr 27, 2026 - 11:47:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 11:47:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 11:49:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.810261 38.884861 38.730381 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.110840 67.400391 67.007812 +util 99.000000 100.000000 96.000000 +[Apr 27, 2026 - 11:49:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 83 : avg iteration time 81.23 seconds +[Apr 27, 2026 - 11:49:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 83 : data loading time 0.00 +[Apr 27, 2026 - 11:49:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 83 : avg forward pass time 20.59 +[Apr 27, 2026 - 11:49:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 83 : backward pass time 18.85 +[Apr 27, 2026 - 11:49:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 83 : optimizer step time 2.38 +[Apr 27, 2026 - 11:49:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 83-------------------- +[Apr 27, 2026 - 11:49:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0093 iter count: 1.0 +[Apr 27, 2026 - 11:49:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[Apr 27, 2026 - 11:49:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 11:49:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.05 seconds +[Apr 27, 2026 - 11:50:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.810152 38.884884 38.730404 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.086426 67.380859 66.968750 +util 99.000000 100.000000 96.000000 +[Apr 27, 2026 - 11:50:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 84 : avg iteration time 81.27 seconds +[Apr 27, 2026 - 11:50:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 84 : data loading time 0.00 +[Apr 27, 2026 - 11:50:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 84 : avg forward pass time 21.08 +[Apr 27, 2026 - 11:50:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 84 : backward pass time 18.16 +[Apr 27, 2026 - 11:50:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 84 : optimizer step time 2.49 +[Apr 27, 2026 - 11:50:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 84-------------------- +[Apr 27, 2026 - 11:50:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0090 iter count: 1.0 +[Apr 27, 2026 - 11:50:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[Apr 27, 2026 - 11:50:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 11:50:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 11:55:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.809750 38.884842 38.730396 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.031250 133.191406 132.605469 +util 72.000000 79.000000 67.000000 +[Apr 27, 2026 - 11:55:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 85 : avg iteration time 273.92 seconds +[Apr 27, 2026 - 11:55:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 85 : data loading time 0.00 +[Apr 27, 2026 - 11:55:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 85 : avg forward pass time 121.92 +[Apr 27, 2026 - 11:55:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 85 : backward pass time 14.99 +[Apr 27, 2026 - 11:55:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 85 : optimizer step time 0.34 +[Apr 27, 2026 - 11:55:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 85-------------------- +[Apr 27, 2026 - 11:55:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 74201.0925 iter count: 1.0 +[Apr 27, 2026 - 11:55:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7358 iter count: 1.0 +[Apr 27, 2026 - 11:55:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 74201.0925iter count: 1.0 +[Apr 27, 2026 - 11:55:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 11:55:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.1243iter count: 1.0 +[Apr 27, 2026 - 11:55:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.7073 iter count: 1.0 +[Apr 27, 2026 - 11:55:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.4991 iter count: 1.0 +[Apr 27, 2026 - 11:55:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 137068.9062iter count: 1.0 +[Apr 27, 2026 - 11:55:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 30.4037iter count: 1.0 +[Apr 27, 2026 - 11:55:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 3262922.0000iter count: 1.0 +[Apr 27, 2026 - 11:55:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7358iter count: 1.0 +[Apr 27, 2026 - 11:55:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.3005 iter count: 1.0 +[Apr 27, 2026 - 11:55:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 40172.3047iter count: 1.0 +[Apr 27, 2026 - 11:55:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 2.9524 iter count: 1.0 +[Apr 27, 2026 - 11:55:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 366.7758iter count: 1.0 +[Apr 27, 2026 - 11:55:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 1.7105 iter count: 1.0 +[Apr 27, 2026 - 11:55:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 30.5995 iter count: 1.0 +[Apr 27, 2026 - 11:55:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.8787 iter count: 1.0 +[Apr 27, 2026 - 11:55:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 17279.9531iter count: 1.0 +[Apr 27, 2026 - 11:55:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[Apr 27, 2026 - 11:55:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 11:55:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 11:56:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.809811 38.884861 38.730396 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.096191 67.419922 66.988281 +util 97.750000 100.000000 92.000000 +[Apr 27, 2026 - 11:56:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 86 : avg iteration time 82.94 seconds +[Apr 27, 2026 - 11:56:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 86 : data loading time 0.00 +[Apr 27, 2026 - 11:56:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 86 : avg forward pass time 21.78 +[Apr 27, 2026 - 11:56:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 86 : backward pass time 18.49 +[Apr 27, 2026 - 11:56:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 86 : optimizer step time 2.19 +[Apr 27, 2026 - 11:56:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 86-------------------- +[Apr 27, 2026 - 11:56:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0085 iter count: 1.0 +[Apr 27, 2026 - 11:56:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[Apr 27, 2026 - 11:56:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 11:56:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.05 seconds +[Apr 27, 2026 - 11:57:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.809812 38.884865 38.730396 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.115723 67.458984 66.968750 +util 99.250000 100.000000 97.000000 +[Apr 27, 2026 - 11:57:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 87 : avg iteration time 81.75 seconds +[Apr 27, 2026 - 11:57:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 87 : data loading time 0.00 +[Apr 27, 2026 - 11:57:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 87 : avg forward pass time 20.66 +[Apr 27, 2026 - 11:57:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 87 : backward pass time 18.72 +[Apr 27, 2026 - 11:57:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 87 : optimizer step time 2.60 +[Apr 27, 2026 - 11:57:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 87-------------------- +[Apr 27, 2026 - 11:57:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[Apr 27, 2026 - 11:57:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[Apr 27, 2026 - 11:57:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 11:57:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 11:59:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.809812 38.884857 38.730404 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.066895 67.205078 66.988281 +util 98.750000 100.000000 95.000000 +[Apr 27, 2026 - 11:59:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 88 : avg iteration time 81.40 seconds +[Apr 27, 2026 - 11:59:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 88 : data loading time 0.00 +[Apr 27, 2026 - 11:59:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 88 : avg forward pass time 20.71 +[Apr 27, 2026 - 11:59:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 88 : backward pass time 19.25 +[Apr 27, 2026 - 11:59:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 88 : optimizer step time 2.33 +[Apr 27, 2026 - 11:59:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 88-------------------- +[Apr 27, 2026 - 11:59:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[Apr 27, 2026 - 11:59:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[Apr 27, 2026 - 11:59:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 11:59:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.05 seconds +[Apr 27, 2026 - 12:00:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.809816 38.884865 38.730396 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.105957 67.400391 66.988281 +util 98.500000 100.000000 95.000000 +[Apr 27, 2026 - 12:00:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 89 : avg iteration time 81.12 seconds +[Apr 27, 2026 - 12:00:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 89 : data loading time 0.00 +[Apr 27, 2026 - 12:00:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 89 : avg forward pass time 20.95 +[Apr 27, 2026 - 12:00:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 89 : backward pass time 18.50 +[Apr 27, 2026 - 12:00:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 89 : optimizer step time 2.24 +[Apr 27, 2026 - 12:00:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 89-------------------- +[Apr 27, 2026 - 12:00:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0088 iter count: 1.0 +[Apr 27, 2026 - 12:00:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[Apr 27, 2026 - 12:00:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 12:00:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.05 seconds +[Apr 27, 2026 - 12:05:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.809658 38.884918 38.730453 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.041992 133.195312 132.625000 +util 65.500000 72.000000 57.000000 +[Apr 27, 2026 - 12:05:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 90 : avg iteration time 278.16 seconds +[Apr 27, 2026 - 12:05:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 90 : data loading time 0.00 +[Apr 27, 2026 - 12:05:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 90 : avg forward pass time 123.84 +[Apr 27, 2026 - 12:05:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 90 : backward pass time 14.70 +[Apr 27, 2026 - 12:05:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 90 : optimizer step time 0.36 +[Apr 27, 2026 - 12:05:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 90-------------------- +[Apr 27, 2026 - 12:05:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 143645.0837 iter count: 1.0 +[Apr 27, 2026 - 12:05:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7277 iter count: 1.0 +[Apr 27, 2026 - 12:05:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 143645.0837iter count: 1.0 +[Apr 27, 2026 - 12:05:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 12:05:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.2521iter count: 1.0 +[Apr 27, 2026 - 12:05:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.9227 iter count: 1.0 +[Apr 27, 2026 - 12:05:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.7410 iter count: 1.0 +[Apr 27, 2026 - 12:05:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 244488.2188iter count: 1.0 +[Apr 27, 2026 - 12:05:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 6.3300 iter count: 1.0 +[Apr 27, 2026 - 12:05:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 5293922.5000iter count: 1.0 +[Apr 27, 2026 - 12:05:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7277iter count: 1.0 +[Apr 27, 2026 - 12:05:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 2.4786 iter count: 1.0 +[Apr 27, 2026 - 12:05:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 142.1855iter count: 1.0 +[Apr 27, 2026 - 12:05:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.9820 iter count: 1.0 +[Apr 27, 2026 - 12:05:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 156988.2656iter count: 1.0 +[Apr 27, 2026 - 12:05:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.1617 iter count: 1.0 +[Apr 27, 2026 - 12:05:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 557.4729iter count: 1.0 +[Apr 27, 2026 - 12:05:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.1196 iter count: 1.0 +[Apr 27, 2026 - 12:05:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 27978.1895iter count: 1.0 +[Apr 27, 2026 - 12:05:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[Apr 27, 2026 - 12:05:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 12:05:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.05 seconds +[Apr 27, 2026 - 12:06:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.809978 38.884895 38.730453 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 68.697754 70.210938 66.929688 +util 97.500000 100.000000 95.000000 +[Apr 27, 2026 - 12:06:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 91 : avg iteration time 76.62 seconds +[Apr 27, 2026 - 12:06:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 91 : data loading time 0.00 +[Apr 27, 2026 - 12:06:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 91 : avg forward pass time 18.87 +[Apr 27, 2026 - 12:06:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 91 : backward pass time 18.34 +[Apr 27, 2026 - 12:06:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 91 : optimizer step time 2.32 +[Apr 27, 2026 - 12:06:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 91-------------------- +[Apr 27, 2026 - 12:06:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0097 iter count: 1.0 +[Apr 27, 2026 - 12:06:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[Apr 27, 2026 - 12:06:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 12:06:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 12:07:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.809951 38.884895 38.730453 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 68.741699 70.250000 67.027344 +util 96.250000 100.000000 91.000000 +[Apr 27, 2026 - 12:07:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 92 : avg iteration time 85.34 seconds +[Apr 27, 2026 - 12:07:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 92 : data loading time 0.00 +[Apr 27, 2026 - 12:07:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 92 : avg forward pass time 23.09 +[Apr 27, 2026 - 12:07:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 92 : backward pass time 18.40 +[Apr 27, 2026 - 12:07:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 92 : optimizer step time 2.59 +[Apr 27, 2026 - 12:07:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 92-------------------- +[Apr 27, 2026 - 12:07:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[Apr 27, 2026 - 12:07:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[Apr 27, 2026 - 12:07:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 12:07:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 12:09:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.809951 38.884895 38.730453 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.311035 67.458984 67.027344 +util 96.500000 100.000000 93.000000 +[Apr 27, 2026 - 12:09:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 93 : avg iteration time 80.87 seconds +[Apr 27, 2026 - 12:09:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 93 : data loading time 0.00 +[Apr 27, 2026 - 12:09:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 93 : avg forward pass time 20.86 +[Apr 27, 2026 - 12:09:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 93 : backward pass time 18.33 +[Apr 27, 2026 - 12:09:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 93 : optimizer step time 2.34 +[Apr 27, 2026 - 12:09:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 93-------------------- +[Apr 27, 2026 - 12:09:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0087 iter count: 1.0 +[Apr 27, 2026 - 12:09:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[Apr 27, 2026 - 12:09:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 12:09:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.05 seconds +[Apr 27, 2026 - 12:10:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.810014 38.884895 38.730457 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.218262 67.478516 67.027344 +util 95.000000 100.000000 91.000000 +[Apr 27, 2026 - 12:10:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 94 : avg iteration time 76.55 seconds +[Apr 27, 2026 - 12:10:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 94 : data loading time 0.00 +[Apr 27, 2026 - 12:10:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 94 : avg forward pass time 18.15 +[Apr 27, 2026 - 12:10:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 94 : backward pass time 18.53 +[Apr 27, 2026 - 12:10:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 94 : optimizer step time 2.57 +[Apr 27, 2026 - 12:10:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 94-------------------- +[Apr 27, 2026 - 12:10:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0092 iter count: 1.0 +[Apr 27, 2026 - 12:10:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[Apr 27, 2026 - 12:10:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 12:10:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 12:14:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.809934 38.884895 38.730457 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.118652 133.152344 133.076172 +util 72.250000 79.000000 68.000000 +[Apr 27, 2026 - 12:14:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 95 : avg iteration time 240.47 seconds +[Apr 27, 2026 - 12:14:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 95 : data loading time 0.00 +[Apr 27, 2026 - 12:14:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 95 : avg forward pass time 108.15 +[Apr 27, 2026 - 12:14:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 95 : backward pass time 8.39 +[Apr 27, 2026 - 12:14:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 95 : optimizer step time 0.33 +[Apr 27, 2026 - 12:14:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 95-------------------- +[Apr 27, 2026 - 12:14:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 809520.1401 iter count: 1.0 +[Apr 27, 2026 - 12:14:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7864 iter count: 1.0 +[Apr 27, 2026 - 12:14:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 809520.1401iter count: 1.0 +[Apr 27, 2026 - 12:14:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 12:14:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.3039iter count: 1.0 +[Apr 27, 2026 - 12:14:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.9486 iter count: 1.0 +[Apr 27, 2026 - 12:14:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.5474 iter count: 1.0 +[Apr 27, 2026 - 12:14:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 894593.3750iter count: 1.0 +[Apr 27, 2026 - 12:14:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 6.6670 iter count: 1.0 +[Apr 27, 2026 - 12:14:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 26558088.0000iter count: 1.0 +[Apr 27, 2026 - 12:14:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7864iter count: 1.0 +[Apr 27, 2026 - 12:14:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 1.7023 iter count: 1.0 +[Apr 27, 2026 - 12:14:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 30.1033 iter count: 1.0 +[Apr 27, 2026 - 12:14:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.0594 iter count: 1.0 +[Apr 27, 2026 - 12:14:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 454.3180iter count: 1.0 +[Apr 27, 2026 - 12:14:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.1813 iter count: 1.0 +[Apr 27, 2026 - 12:14:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 579.7652iter count: 1.0 +[Apr 27, 2026 - 12:14:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.3657 iter count: 1.0 +[Apr 27, 2026 - 12:14:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 45769.0430iter count: 1.0 +[Apr 27, 2026 - 12:14:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[Apr 27, 2026 - 12:14:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 12:14:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 12:15:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.809998 38.884895 38.730457 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.105957 67.185547 67.027344 +util 98.000000 100.000000 94.000000 +[Apr 27, 2026 - 12:15:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 96 : avg iteration time 49.76 seconds +[Apr 27, 2026 - 12:15:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 96 : data loading time 0.00 +[Apr 27, 2026 - 12:15:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 96 : avg forward pass time 12.24 +[Apr 27, 2026 - 12:15:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 96 : backward pass time 12.15 +[Apr 27, 2026 - 12:15:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 96 : optimizer step time 1.55 +[Apr 27, 2026 - 12:15:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 96-------------------- +[Apr 27, 2026 - 12:15:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0096 iter count: 1.0 +[Apr 27, 2026 - 12:15:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[Apr 27, 2026 - 12:15:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 12:15:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 12:16:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.810311 38.884907 38.730457 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.145020 67.380859 67.027344 +util 98.500000 100.000000 96.000000 +[Apr 27, 2026 - 12:16:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 97 : avg iteration time 50.30 seconds +[Apr 27, 2026 - 12:16:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 97 : data loading time 0.00 +[Apr 27, 2026 - 12:16:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 97 : avg forward pass time 12.58 +[Apr 27, 2026 - 12:16:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 97 : backward pass time 12.57 +[Apr 27, 2026 - 12:16:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 97 : optimizer step time 1.02 +[Apr 27, 2026 - 12:16:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 97-------------------- +[Apr 27, 2026 - 12:16:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0104 iter count: 1.0 +[Apr 27, 2026 - 12:16:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[Apr 27, 2026 - 12:16:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 12:16:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 12:17:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.810326 38.884895 38.730457 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.135254 67.380859 66.988281 +util 97.250000 99.000000 95.000000 +[Apr 27, 2026 - 12:17:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 98 : avg iteration time 49.11 seconds +[Apr 27, 2026 - 12:17:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 98 : data loading time 0.00 +[Apr 27, 2026 - 12:17:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 98 : avg forward pass time 12.50 +[Apr 27, 2026 - 12:17:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 98 : backward pass time 11.55 +[Apr 27, 2026 - 12:17:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 98 : optimizer step time 0.98 +[Apr 27, 2026 - 12:17:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 98-------------------- +[Apr 27, 2026 - 12:17:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0105 iter count: 1.0 +[Apr 27, 2026 - 12:17:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[Apr 27, 2026 - 12:17:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 12:17:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 12:17:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.810227 38.884895 38.730457 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.086426 67.185547 66.988281 +util 98.000000 99.000000 96.000000 +[Apr 27, 2026 - 12:17:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 99 : avg iteration time 50.22 seconds +[Apr 27, 2026 - 12:17:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 99 : data loading time 0.00 +[Apr 27, 2026 - 12:17:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 99 : avg forward pass time 12.55 +[Apr 27, 2026 - 12:17:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 99 : backward pass time 11.52 +[Apr 27, 2026 - 12:17:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 99 : optimizer step time 0.98 +[Apr 27, 2026 - 12:17:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 99-------------------- +[Apr 27, 2026 - 12:17:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0104 iter count: 1.0 +[Apr 27, 2026 - 12:17:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[Apr 27, 2026 - 12:17:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 12:17:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 12:20:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.810116 38.884895 38.730473 +peak_gpu_mem_gb 129.259261 129.260806 129.258746 +peak_gpu_mem_reserved_gb 133.154297 133.193359 133.076172 +util 71.250000 75.000000 66.000000 +[Apr 27, 2026 - 12:20:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 100 : avg iteration time 180.46 seconds +[Apr 27, 2026 - 12:20:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 100 : data loading time 0.00 +[Apr 27, 2026 - 12:20:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 100 : avg forward pass time 80.71 +[Apr 27, 2026 - 12:20:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 100 : backward pass time 8.40 +[Apr 27, 2026 - 12:20:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 100 : optimizer step time 0.34 +[Apr 27, 2026 - 12:20:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 100-------------------- +[Apr 27, 2026 - 12:20:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 62780.5684 iter count: 1.0 +[Apr 27, 2026 - 12:20:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6926 iter count: 1.0 +[Apr 27, 2026 - 12:20:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 62780.5684iter count: 1.0 +[Apr 27, 2026 - 12:20:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 12:20:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.1584iter count: 1.0 +[Apr 27, 2026 - 12:20:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.7669 iter count: 1.0 +[Apr 27, 2026 - 12:20:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.0700 iter count: 1.0 +[Apr 27, 2026 - 12:20:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 101429.8438iter count: 1.0 +[Apr 27, 2026 - 12:20:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 4.6353 iter count: 1.0 +[Apr 27, 2026 - 12:20:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 1383252.1250iter count: 1.0 +[Apr 27, 2026 - 12:20:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6926iter count: 1.0 +[Apr 27, 2026 - 12:20:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.8883 iter count: 1.0 +[Apr 27, 2026 - 12:20:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 2384.3047iter count: 1.0 +[Apr 27, 2026 - 12:20:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.4290 iter count: 1.0 +[Apr 27, 2026 - 12:20:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 51945.8906iter count: 1.0 +[Apr 27, 2026 - 12:20:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.6700 iter count: 1.0 +[Apr 27, 2026 - 12:20:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 11383.5635iter count: 1.0 +[Apr 27, 2026 - 12:20:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.2821 iter count: 1.0 +[Apr 27, 2026 - 12:20:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 38724.3633iter count: 1.0 +[Apr 27, 2026 - 12:20:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0120 iter count: 1.0 +[Apr 27, 2026 - 12:20:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[Apr 27, 2026 - 12:21:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 18.20 seconds +[Apr 27, 2026 - 12:21:11 | INFO | fastgen.trainer:save_checkpoint:266 ] Saving checkpoint iteration 100 +[Apr 27, 2026 - 12:21:11 | INFO | fastgen.utils.checkpointer:save:301 ] Saving FSDP model to prefix /tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000100 +[Apr 27, 2026 - 12:21:27 | SUCCESS | fastgen.utils.checkpointer:save:329 ] Model saved at iteration 100 +[Apr 27, 2026 - 12:21:27 | INFO | fastgen.trainer:validate:358 ] Validation iteration 100 +[Apr 27, 2026 - 12:21:28 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 12:21:30 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 2.0s +[Apr 27, 2026 - 12:21:33 | INFO | fastgen.callbacks.wandb:_get_syncnet_scorer:402 ] [WandbCallback] Loaded SyncCScorer from /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model +[Apr 27, 2026 - 12:21:33 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 0: sync_c=6.119 +[Apr 27, 2026 - 12:21:34 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 12:21:36 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.8s +[Apr 27, 2026 - 12:21:39 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 1: sync_c=5.930 +[Apr 27, 2026 - 12:21:39 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 12:21:41 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.8s +[Apr 27, 2026 - 12:21:44 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 2: sync_c=4.672 +[Apr 27, 2026 - 12:21:44 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 12:21:46 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.8s +[Apr 27, 2026 - 12:21:49 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 3: sync_c=3.948 +[Apr 27, 2026 - 12:21:49 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 12:21:51 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.8s +[Apr 27, 2026 - 12:21:54 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 4: sync_c=5.118 +[Apr 27, 2026 - 12:21:54 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 12:21:56 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.8s +[Apr 27, 2026 - 12:21:59 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 5: sync_c=6.354 +[Apr 27, 2026 - 12:21:59 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 12:22:01 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.8s +[Apr 27, 2026 - 12:22:04 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 6: sync_c=5.878 +[Apr 27, 2026 - 12:22:04 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 12:22:06 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.8s +[Apr 27, 2026 - 12:22:09 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 7: sync_c=4.747 +[Apr 27, 2026 - 12:22:10 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 12:22:11 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.8s +[Apr 27, 2026 - 12:22:14 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 8: sync_c=5.557 +[Apr 27, 2026 - 12:22:15 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 12:22:16 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.8s +[Apr 27, 2026 - 12:22:19 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 9: sync_c=2.502 +[Apr 27, 2026 - 12:22:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging val0 stats at iteration 100-------------------- +[Apr 27, 2026 - 12:22:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0000 iter count: 10.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[Apr 27, 2026 - 12:22:37 | INFO | fastgen.callbacks.wandb:on_validation_end:763 ] Logged 10 val videos at iteration 100 +[Apr 27, 2026 - 12:22:37 | INFO | fastgen.callbacks.wandb:on_validation_end:769 ] [SyncEval] val0 mean sync_c: 5.082 (n=10) +[Apr 27, 2026 - 12:23:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.075908 43.159142 42.985947 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.855469 70.332031 69.566406 +util 96.750000 100.000000 94.000000 +[Apr 27, 2026 - 12:23:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 101 : avg iteration time 150.91 seconds +[Apr 27, 2026 - 12:23:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 101 : data loading time 0.00 +[Apr 27, 2026 - 12:23:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 101 : avg forward pass time 11.00 +[Apr 27, 2026 - 12:23:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 101 : backward pass time 11.93 +[Apr 27, 2026 - 12:23:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 101 : optimizer step time 0.98 +[Apr 27, 2026 - 12:23:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 101-------------------- +[Apr 27, 2026 - 12:23:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0129 iter count: 1.0 +[Apr 27, 2026 - 12:23:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0129 iter count: 1.0 +[Apr 27, 2026 - 12:23:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 12:23:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 12:24:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.075918 43.159142 42.985970 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.796875 70.195312 69.546875 +util 98.000000 99.000000 95.000000 +[Apr 27, 2026 - 12:24:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 102 : avg iteration time 52.69 seconds +[Apr 27, 2026 - 12:24:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 102 : data loading time 0.00 +[Apr 27, 2026 - 12:24:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 102 : avg forward pass time 14.09 +[Apr 27, 2026 - 12:24:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 102 : backward pass time 11.60 +[Apr 27, 2026 - 12:24:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 102 : optimizer step time 0.97 +[Apr 27, 2026 - 12:24:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 102-------------------- +[Apr 27, 2026 - 12:24:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0149 iter count: 1.0 +[Apr 27, 2026 - 12:24:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0149 iter count: 1.0 +[Apr 27, 2026 - 12:24:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 12:24:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 12:25:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.075475 43.159164 42.985947 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.831055 70.312500 69.527344 +util 96.500000 100.000000 93.000000 +[Apr 27, 2026 - 12:25:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 103 : avg iteration time 47.47 seconds +[Apr 27, 2026 - 12:25:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 103 : data loading time 0.00 +[Apr 27, 2026 - 12:25:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 103 : avg forward pass time 11.28 +[Apr 27, 2026 - 12:25:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 103 : backward pass time 11.59 +[Apr 27, 2026 - 12:25:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 103 : optimizer step time 0.98 +[Apr 27, 2026 - 12:25:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 103-------------------- +[Apr 27, 2026 - 12:25:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0123 iter count: 1.0 +[Apr 27, 2026 - 12:25:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0123 iter count: 1.0 +[Apr 27, 2026 - 12:25:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 12:25:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 12:25:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.075505 43.159286 42.985947 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.806641 70.234375 69.546875 +util 98.250000 100.000000 95.000000 +[Apr 27, 2026 - 12:25:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 104 : avg iteration time 50.34 seconds +[Apr 27, 2026 - 12:25:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 104 : data loading time 0.00 +[Apr 27, 2026 - 12:25:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 104 : avg forward pass time 12.60 +[Apr 27, 2026 - 12:25:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 104 : backward pass time 12.53 +[Apr 27, 2026 - 12:25:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 104 : optimizer step time 1.01 +[Apr 27, 2026 - 12:25:54 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 104-------------------- +[Apr 27, 2026 - 12:25:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0146 iter count: 1.0 +[Apr 27, 2026 - 12:25:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0146 iter count: 1.0 +[Apr 27, 2026 - 12:25:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 12:25:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 12:28:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076175 43.159737 42.986473 +peak_gpu_mem_gb 131.581049 131.620712 131.567828 +peak_gpu_mem_reserved_gb 135.538086 135.619141 135.498047 +util 65.500000 72.000000 57.000000 +[Apr 27, 2026 - 12:28:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 105 : avg iteration time 174.07 seconds +[Apr 27, 2026 - 12:28:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 105 : data loading time 0.00 +[Apr 27, 2026 - 12:28:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 105 : avg forward pass time 78.25 +[Apr 27, 2026 - 12:28:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 105 : backward pass time 8.87 +[Apr 27, 2026 - 12:28:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 105 : optimizer step time 0.35 +[Apr 27, 2026 - 12:28:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 105-------------------- +[Apr 27, 2026 - 12:28:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 118595.2383 iter count: 1.0 +[Apr 27, 2026 - 12:28:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7291 iter count: 1.0 +[Apr 27, 2026 - 12:28:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 118595.2383iter count: 1.0 +[Apr 27, 2026 - 12:28:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 12:28:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.1081iter count: 1.0 +[Apr 27, 2026 - 12:28:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.7797 iter count: 1.0 +[Apr 27, 2026 - 12:28:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.2074 iter count: 1.0 +[Apr 27, 2026 - 12:28:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 150255.4375iter count: 1.0 +[Apr 27, 2026 - 12:28:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 4.7562 iter count: 1.0 +[Apr 27, 2026 - 12:28:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 1820996.0000iter count: 1.0 +[Apr 27, 2026 - 12:28:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7291iter count: 1.0 +[Apr 27, 2026 - 12:28:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 6.2060 iter count: 1.0 +[Apr 27, 2026 - 12:28:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 245748.8906iter count: 1.0 +[Apr 27, 2026 - 12:28:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.0240 iter count: 1.0 +[Apr 27, 2026 - 12:28:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 23107.7227iter count: 1.0 +[Apr 27, 2026 - 12:28:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 7.0423 iter count: 1.0 +[Apr 27, 2026 - 12:28:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 1308875.1250iter count: 1.0 +[Apr 27, 2026 - 12:28:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 1.1293 iter count: 1.0 +[Apr 27, 2026 - 12:28:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 9.5698 iter count: 1.0 +[Apr 27, 2026 - 12:28:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0124 iter count: 1.0 +[Apr 27, 2026 - 12:28:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 12:28:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.02 seconds +[Apr 27, 2026 - 12:29:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076601 43.159855 42.986469 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.967773 70.015625 69.859375 +util 98.000000 100.000000 94.000000 +[Apr 27, 2026 - 12:29:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 106 : avg iteration time 48.91 seconds +[Apr 27, 2026 - 12:29:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 106 : data loading time 0.00 +[Apr 27, 2026 - 12:29:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 106 : avg forward pass time 11.70 +[Apr 27, 2026 - 12:29:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 106 : backward pass time 11.59 +[Apr 27, 2026 - 12:29:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 106 : optimizer step time 0.97 +[Apr 27, 2026 - 12:29:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 106-------------------- +[Apr 27, 2026 - 12:29:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0121 iter count: 1.0 +[Apr 27, 2026 - 12:29:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[Apr 27, 2026 - 12:29:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 12:29:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 12:30:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076637 43.159878 42.986542 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.972656 70.078125 69.859375 +util 98.000000 100.000000 96.000000 +[Apr 27, 2026 - 12:30:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 107 : avg iteration time 53.96 seconds +[Apr 27, 2026 - 12:30:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 107 : data loading time 0.00 +[Apr 27, 2026 - 12:30:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 107 : avg forward pass time 14.17 +[Apr 27, 2026 - 12:30:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 107 : backward pass time 12.59 +[Apr 27, 2026 - 12:30:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 107 : optimizer step time 1.41 +[Apr 27, 2026 - 12:30:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 107-------------------- +[Apr 27, 2026 - 12:30:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0128 iter count: 1.0 +[Apr 27, 2026 - 12:30:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0128 iter count: 1.0 +[Apr 27, 2026 - 12:30:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 12:30:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 12:31:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076621 43.159855 42.986519 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.001953 70.015625 69.960938 +util 94.250000 99.000000 92.000000 +[Apr 27, 2026 - 12:31:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 108 : avg iteration time 52.74 seconds +[Apr 27, 2026 - 12:31:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 108 : data loading time 0.00 +[Apr 27, 2026 - 12:31:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 108 : avg forward pass time 14.30 +[Apr 27, 2026 - 12:31:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 108 : backward pass time 11.53 +[Apr 27, 2026 - 12:31:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 108 : optimizer step time 1.02 +[Apr 27, 2026 - 12:31:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 108-------------------- +[Apr 27, 2026 - 12:31:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0117 iter count: 1.0 +[Apr 27, 2026 - 12:31:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[Apr 27, 2026 - 12:31:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 12:31:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 12:32:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076658 43.159878 42.986542 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.933594 69.976562 69.839844 +util 97.500000 100.000000 92.000000 +[Apr 27, 2026 - 12:32:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 109 : avg iteration time 50.49 seconds +[Apr 27, 2026 - 12:32:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 109 : data loading time 0.00 +[Apr 27, 2026 - 12:32:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 109 : avg forward pass time 12.60 +[Apr 27, 2026 - 12:32:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 109 : backward pass time 11.59 +[Apr 27, 2026 - 12:32:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 109 : optimizer step time 0.97 +[Apr 27, 2026 - 12:32:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 109-------------------- +[Apr 27, 2026 - 12:32:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0116 iter count: 1.0 +[Apr 27, 2026 - 12:32:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[Apr 27, 2026 - 12:32:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 12:32:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 12:35:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076081 43.159855 42.986523 +peak_gpu_mem_gb 131.581049 131.620712 131.567828 +peak_gpu_mem_reserved_gb 135.542969 135.580078 135.478516 +util 72.250000 77.000000 65.000000 +[Apr 27, 2026 - 12:35:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 110 : avg iteration time 175.41 seconds +[Apr 27, 2026 - 12:35:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 110 : data loading time 0.00 +[Apr 27, 2026 - 12:35:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 110 : avg forward pass time 78.58 +[Apr 27, 2026 - 12:35:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 110 : backward pass time 9.63 +[Apr 27, 2026 - 12:35:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 110 : optimizer step time 0.34 +[Apr 27, 2026 - 12:35:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 110-------------------- +[Apr 27, 2026 - 12:35:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 161739.4766 iter count: 1.0 +[Apr 27, 2026 - 12:35:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7067 iter count: 1.0 +[Apr 27, 2026 - 12:35:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 161739.4766iter count: 1.0 +[Apr 27, 2026 - 12:35:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 12:35:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.7191iter count: 1.0 +[Apr 27, 2026 - 12:35:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.6945 iter count: 1.0 +[Apr 27, 2026 - 12:35:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.3433 iter count: 1.0 +[Apr 27, 2026 - 12:35:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 309682.2812iter count: 1.0 +[Apr 27, 2026 - 12:35:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 29.6353iter count: 1.0 +[Apr 27, 2026 - 12:35:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 2389450.5000iter count: 1.0 +[Apr 27, 2026 - 12:35:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7067iter count: 1.0 +[Apr 27, 2026 - 12:35:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.7810 iter count: 1.0 +[Apr 27, 2026 - 12:35:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 14213.1113iter count: 1.0 +[Apr 27, 2026 - 12:35:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.6265 iter count: 1.0 +[Apr 27, 2026 - 12:35:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 10436.4258iter count: 1.0 +[Apr 27, 2026 - 12:35:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.3956 iter count: 1.0 +[Apr 27, 2026 - 12:35:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 889.8932iter count: 1.0 +[Apr 27, 2026 - 12:35:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.4575 iter count: 1.0 +[Apr 27, 2026 - 12:35:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 406362.9062iter count: 1.0 +[Apr 27, 2026 - 12:35:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0126 iter count: 1.0 +[Apr 27, 2026 - 12:35:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 12:35:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 12:36:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076108 43.159874 42.986568 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.860352 70.035156 69.585938 +util 98.000000 99.000000 95.000000 +[Apr 27, 2026 - 12:36:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 111 : avg iteration time 51.62 seconds +[Apr 27, 2026 - 12:36:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 111 : data loading time 0.00 +[Apr 27, 2026 - 12:36:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 111 : avg forward pass time 13.75 +[Apr 27, 2026 - 12:36:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 111 : backward pass time 11.57 +[Apr 27, 2026 - 12:36:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 111 : optimizer step time 0.99 +[Apr 27, 2026 - 12:36:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 111-------------------- +[Apr 27, 2026 - 12:36:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0113 iter count: 1.0 +[Apr 27, 2026 - 12:36:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[Apr 27, 2026 - 12:36:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 12:36:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 12:36:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076381 43.159893 42.986523 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.850586 70.093750 69.566406 +util 96.500000 100.000000 90.000000 +[Apr 27, 2026 - 12:36:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 112 : avg iteration time 49.82 seconds +[Apr 27, 2026 - 12:36:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 112 : data loading time 0.00 +[Apr 27, 2026 - 12:36:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 112 : avg forward pass time 12.70 +[Apr 27, 2026 - 12:36:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 112 : backward pass time 11.55 +[Apr 27, 2026 - 12:36:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 112 : optimizer step time 0.98 +[Apr 27, 2026 - 12:36:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 112-------------------- +[Apr 27, 2026 - 12:36:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0122 iter count: 1.0 +[Apr 27, 2026 - 12:36:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0122 iter count: 1.0 +[Apr 27, 2026 - 12:36:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 12:36:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 12:37:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076569 43.159855 42.986568 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.708984 69.800781 69.507812 +util 95.000000 99.000000 92.000000 +[Apr 27, 2026 - 12:37:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 113 : avg iteration time 47.27 seconds +[Apr 27, 2026 - 12:37:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 113 : data loading time 0.00 +[Apr 27, 2026 - 12:37:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 113 : avg forward pass time 11.21 +[Apr 27, 2026 - 12:37:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 113 : backward pass time 11.51 +[Apr 27, 2026 - 12:37:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 113 : optimizer step time 1.03 +[Apr 27, 2026 - 12:37:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 113-------------------- +[Apr 27, 2026 - 12:37:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0121 iter count: 1.0 +[Apr 27, 2026 - 12:37:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[Apr 27, 2026 - 12:37:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 12:37:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 12:38:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076573 43.159855 42.986530 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.870117 69.937500 69.746094 +util 96.000000 99.000000 93.000000 +[Apr 27, 2026 - 12:38:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 114 : avg iteration time 47.61 seconds +[Apr 27, 2026 - 12:38:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 114 : data loading time 0.00 +[Apr 27, 2026 - 12:38:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 114 : avg forward pass time 11.07 +[Apr 27, 2026 - 12:38:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 114 : backward pass time 12.83 +[Apr 27, 2026 - 12:38:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 114 : optimizer step time 1.08 +[Apr 27, 2026 - 12:38:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 114-------------------- +[Apr 27, 2026 - 12:38:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0121 iter count: 1.0 +[Apr 27, 2026 - 12:38:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[Apr 27, 2026 - 12:38:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 12:38:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 12:41:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076036 43.159904 42.986523 +peak_gpu_mem_gb 131.581049 131.620712 131.567828 +peak_gpu_mem_reserved_gb 135.547852 135.580078 135.458984 +util 68.500000 75.000000 64.000000 +[Apr 27, 2026 - 12:41:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 115 : avg iteration time 176.61 seconds +[Apr 27, 2026 - 12:41:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 115 : data loading time 0.00 +[Apr 27, 2026 - 12:41:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 115 : avg forward pass time 79.79 +[Apr 27, 2026 - 12:41:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 115 : backward pass time 8.41 +[Apr 27, 2026 - 12:41:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 115 : optimizer step time 0.34 +[Apr 27, 2026 - 12:41:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 115-------------------- +[Apr 27, 2026 - 12:41:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 1141226.6091 iter count: 1.0 +[Apr 27, 2026 - 12:41:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7059 iter count: 1.0 +[Apr 27, 2026 - 12:41:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 1141226.6091iter count: 1.0 +[Apr 27, 2026 - 12:41:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 12:41:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.5838iter count: 1.0 +[Apr 27, 2026 - 12:41:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.5191 iter count: 1.0 +[Apr 27, 2026 - 12:41:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.6893 iter count: 1.0 +[Apr 27, 2026 - 12:41:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 1321427.8750iter count: 1.0 +[Apr 27, 2026 - 12:41:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 20.8691iter count: 1.0 +[Apr 27, 2026 - 12:41:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 35274460.0000iter count: 1.0 +[Apr 27, 2026 - 12:41:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7059iter count: 1.0 +[Apr 27, 2026 - 12:41:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 6.4839 iter count: 1.0 +[Apr 27, 2026 - 12:41:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 428394.6562iter count: 1.0 +[Apr 27, 2026 - 12:41:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 7.1977 iter count: 1.0 +[Apr 27, 2026 - 12:41:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 1785950.1250iter count: 1.0 +[Apr 27, 2026 - 12:41:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 2.2523 iter count: 1.0 +[Apr 27, 2026 - 12:41:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 90.4376 iter count: 1.0 +[Apr 27, 2026 - 12:41:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 1.6313 iter count: 1.0 +[Apr 27, 2026 - 12:41:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 26.1183 iter count: 1.0 +[Apr 27, 2026 - 12:41:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0127 iter count: 1.0 +[Apr 27, 2026 - 12:41:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 12:41:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 12:42:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076036 43.159904 42.986523 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.674805 70.097656 69.390625 +util 96.250000 99.000000 91.000000 +[Apr 27, 2026 - 12:42:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 116 : avg iteration time 54.57 seconds +[Apr 27, 2026 - 12:42:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 116 : data loading time 0.00 +[Apr 27, 2026 - 12:42:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 116 : avg forward pass time 14.62 +[Apr 27, 2026 - 12:42:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 116 : backward pass time 11.53 +[Apr 27, 2026 - 12:42:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 116 : optimizer step time 1.20 +[Apr 27, 2026 - 12:42:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 116-------------------- +[Apr 27, 2026 - 12:42:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0120 iter count: 1.0 +[Apr 27, 2026 - 12:42:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0120 iter count: 1.0 +[Apr 27, 2026 - 12:42:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 12:42:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 12:43:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076367 43.159882 42.986523 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.923828 70.039062 69.859375 +util 98.250000 99.000000 97.000000 +[Apr 27, 2026 - 12:43:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 117 : avg iteration time 47.66 seconds +[Apr 27, 2026 - 12:43:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 117 : data loading time 0.00 +[Apr 27, 2026 - 12:43:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 117 : avg forward pass time 11.10 +[Apr 27, 2026 - 12:43:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 117 : backward pass time 12.81 +[Apr 27, 2026 - 12:43:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 117 : optimizer step time 1.08 +[Apr 27, 2026 - 12:43:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 117-------------------- +[Apr 27, 2026 - 12:43:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0126 iter count: 1.0 +[Apr 27, 2026 - 12:43:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0126 iter count: 1.0 +[Apr 27, 2026 - 12:43:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 12:43:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 12:43:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076549 43.159904 42.986546 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.679688 70.019531 69.410156 +util 96.500000 99.000000 94.000000 +[Apr 27, 2026 - 12:43:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 118 : avg iteration time 46.97 seconds +[Apr 27, 2026 - 12:43:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 118 : data loading time 0.00 +[Apr 27, 2026 - 12:43:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 118 : avg forward pass time 11.43 +[Apr 27, 2026 - 12:43:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 118 : backward pass time 11.59 +[Apr 27, 2026 - 12:43:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 118 : optimizer step time 0.98 +[Apr 27, 2026 - 12:43:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 118-------------------- +[Apr 27, 2026 - 12:43:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0124 iter count: 1.0 +[Apr 27, 2026 - 12:43:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0124 iter count: 1.0 +[Apr 27, 2026 - 12:43:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 12:43:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 12:44:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076404 43.159889 42.986523 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.787109 70.039062 69.585938 +util 96.000000 99.000000 93.000000 +[Apr 27, 2026 - 12:44:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 119 : avg iteration time 50.00 seconds +[Apr 27, 2026 - 12:44:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 119 : data loading time 0.00 +[Apr 27, 2026 - 12:44:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 119 : avg forward pass time 12.73 +[Apr 27, 2026 - 12:44:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 119 : backward pass time 11.57 +[Apr 27, 2026 - 12:44:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 119 : optimizer step time 0.98 +[Apr 27, 2026 - 12:44:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 119-------------------- +[Apr 27, 2026 - 12:44:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0137 iter count: 1.0 +[Apr 27, 2026 - 12:44:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0137 iter count: 1.0 +[Apr 27, 2026 - 12:44:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 12:44:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 12:47:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076304 43.159904 42.986542 +peak_gpu_mem_gb 131.581049 131.620712 131.567828 +peak_gpu_mem_reserved_gb 135.547852 135.658203 135.458984 +util 79.750000 95.000000 70.000000 +[Apr 27, 2026 - 12:47:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 120 : avg iteration time 177.64 seconds +[Apr 27, 2026 - 12:47:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 120 : data loading time 0.00 +[Apr 27, 2026 - 12:47:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 120 : avg forward pass time 79.74 +[Apr 27, 2026 - 12:47:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 120 : backward pass time 9.09 +[Apr 27, 2026 - 12:47:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 120 : optimizer step time 0.34 +[Apr 27, 2026 - 12:47:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 120-------------------- +[Apr 27, 2026 - 12:47:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 598304.3662 iter count: 1.0 +[Apr 27, 2026 - 12:47:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7061 iter count: 1.0 +[Apr 27, 2026 - 12:47:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 598304.3662iter count: 1.0 +[Apr 27, 2026 - 12:47:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 12:47:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.4538iter count: 1.0 +[Apr 27, 2026 - 12:47:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.0130 iter count: 1.0 +[Apr 27, 2026 - 12:47:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.3840 iter count: 1.0 +[Apr 27, 2026 - 12:47:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 676509.3750iter count: 1.0 +[Apr 27, 2026 - 12:47:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 7.5840 iter count: 1.0 +[Apr 27, 2026 - 12:47:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 19151926.0000iter count: 1.0 +[Apr 27, 2026 - 12:47:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7061iter count: 1.0 +[Apr 27, 2026 - 12:47:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.3456 iter count: 1.0 +[Apr 27, 2026 - 12:47:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 5950.8423iter count: 1.0 +[Apr 27, 2026 - 12:47:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 2.0454 iter count: 1.0 +[Apr 27, 2026 - 12:47:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 59.7913 iter count: 1.0 +[Apr 27, 2026 - 12:47:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.6757 iter count: 1.0 +[Apr 27, 2026 - 12:47:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 11515.1357iter count: 1.0 +[Apr 27, 2026 - 12:47:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.1310 iter count: 1.0 +[Apr 27, 2026 - 12:47:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 3874.0227iter count: 1.0 +[Apr 27, 2026 - 12:47:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[Apr 27, 2026 - 12:47:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 12:47:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 12:48:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076313 43.159904 42.986557 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 71.193359 72.769531 69.351562 +util 96.750000 99.000000 91.000000 +[Apr 27, 2026 - 12:48:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 121 : avg iteration time 50.08 seconds +[Apr 27, 2026 - 12:48:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 121 : data loading time 0.00 +[Apr 27, 2026 - 12:48:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 121 : avg forward pass time 12.20 +[Apr 27, 2026 - 12:48:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 121 : backward pass time 12.97 +[Apr 27, 2026 - 12:48:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 121 : optimizer step time 1.11 +[Apr 27, 2026 - 12:48:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 121-------------------- +[Apr 27, 2026 - 12:48:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0150 iter count: 1.0 +[Apr 27, 2026 - 12:48:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0150 iter count: 1.0 +[Apr 27, 2026 - 12:48:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 12:48:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 12:49:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076407 43.159904 42.986542 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 72.042969 72.792969 69.839844 +util 96.000000 99.000000 92.000000 +[Apr 27, 2026 - 12:49:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 122 : avg iteration time 50.23 seconds +[Apr 27, 2026 - 12:49:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 122 : data loading time 0.00 +[Apr 27, 2026 - 12:49:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 122 : avg forward pass time 13.06 +[Apr 27, 2026 - 12:49:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 122 : backward pass time 11.57 +[Apr 27, 2026 - 12:49:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 122 : optimizer step time 0.98 +[Apr 27, 2026 - 12:49:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 122-------------------- +[Apr 27, 2026 - 12:49:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0132 iter count: 1.0 +[Apr 27, 2026 - 12:49:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0132 iter count: 1.0 +[Apr 27, 2026 - 12:49:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 12:49:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 12:50:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076402 43.159882 42.986542 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 71.334961 72.789062 69.839844 +util 97.000000 99.000000 92.000000 +[Apr 27, 2026 - 12:50:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 123 : avg iteration time 50.70 seconds +[Apr 27, 2026 - 12:50:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 123 : data loading time 0.00 +[Apr 27, 2026 - 12:50:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 123 : avg forward pass time 12.77 +[Apr 27, 2026 - 12:50:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 123 : backward pass time 11.56 +[Apr 27, 2026 - 12:50:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 123 : optimizer step time 0.99 +[Apr 27, 2026 - 12:50:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 123-------------------- +[Apr 27, 2026 - 12:50:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0146 iter count: 1.0 +[Apr 27, 2026 - 12:50:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0146 iter count: 1.0 +[Apr 27, 2026 - 12:50:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 12:50:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 12:51:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076412 43.159904 42.986546 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 72.013672 72.753906 69.839844 +util 96.000000 99.000000 93.000000 +[Apr 27, 2026 - 12:51:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 124 : avg iteration time 50.54 seconds +[Apr 27, 2026 - 12:51:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 124 : data loading time 0.00 +[Apr 27, 2026 - 12:51:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 124 : avg forward pass time 12.66 +[Apr 27, 2026 - 12:51:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 124 : backward pass time 11.56 +[Apr 27, 2026 - 12:51:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 124 : optimizer step time 0.98 +[Apr 27, 2026 - 12:51:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 124-------------------- +[Apr 27, 2026 - 12:51:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0142 iter count: 1.0 +[Apr 27, 2026 - 12:51:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0142 iter count: 1.0 +[Apr 27, 2026 - 12:51:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 12:51:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 12:53:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076206 43.159882 42.986542 +peak_gpu_mem_gb 131.581049 131.620712 131.567828 +peak_gpu_mem_reserved_gb 135.567383 135.638672 135.517578 +util 72.750000 78.000000 67.000000 +[Apr 27, 2026 - 12:53:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 125 : avg iteration time 173.23 seconds +[Apr 27, 2026 - 12:53:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 125 : data loading time 0.00 +[Apr 27, 2026 - 12:53:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 125 : avg forward pass time 77.23 +[Apr 27, 2026 - 12:53:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 125 : backward pass time 10.03 +[Apr 27, 2026 - 12:53:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 125 : optimizer step time 0.34 +[Apr 27, 2026 - 12:53:54 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 125-------------------- +[Apr 27, 2026 - 12:53:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 7220110.2009 iter count: 1.0 +[Apr 27, 2026 - 12:53:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6101 iter count: 1.0 +[Apr 27, 2026 - 12:53:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 7220110.2009iter count: 1.0 +[Apr 27, 2026 - 12:53:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 12:53:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.3423iter count: 1.0 +[Apr 27, 2026 - 12:53:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.2449 iter count: 1.0 +[Apr 27, 2026 - 12:53:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 9.9878 iter count: 1.0 +[Apr 27, 2026 - 12:53:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 15121956.0000iter count: 1.0 +[Apr 27, 2026 - 12:53:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 12.0594iter count: 1.0 +[Apr 27, 2026 - 12:53:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 473517056.0000iter count: 1.0 +[Apr 27, 2026 - 12:53:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6101iter count: 1.0 +[Apr 27, 2026 - 12:53:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.0637 iter count: 1.0 +[Apr 27, 2026 - 12:53:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 3385.7891iter count: 1.0 +[Apr 27, 2026 - 12:53:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.7717 iter count: 1.0 +[Apr 27, 2026 - 12:53:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 1888.0514iter count: 1.0 +[Apr 27, 2026 - 12:53:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 6.1639 iter count: 1.0 +[Apr 27, 2026 - 12:53:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 225894.5625iter count: 1.0 +[Apr 27, 2026 - 12:53:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.3950 iter count: 1.0 +[Apr 27, 2026 - 12:53:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 48529.0977iter count: 1.0 +[Apr 27, 2026 - 12:53:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0131 iter count: 1.0 +[Apr 27, 2026 - 12:53:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 12:53:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 12:54:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076212 43.159904 42.986542 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.699219 70.097656 69.410156 +util 97.250000 99.000000 95.000000 +[Apr 27, 2026 - 12:54:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 126 : avg iteration time 54.60 seconds +[Apr 27, 2026 - 12:54:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 126 : data loading time 0.00 +[Apr 27, 2026 - 12:54:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 126 : avg forward pass time 15.24 +[Apr 27, 2026 - 12:54:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 126 : backward pass time 11.52 +[Apr 27, 2026 - 12:54:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 126 : optimizer step time 1.01 +[Apr 27, 2026 - 12:54:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 126-------------------- +[Apr 27, 2026 - 12:54:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0127 iter count: 1.0 +[Apr 27, 2026 - 12:54:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0127 iter count: 1.0 +[Apr 27, 2026 - 12:54:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 12:54:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 12:55:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076206 43.159882 42.986542 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.796875 70.156250 69.566406 +util 97.250000 99.000000 92.000000 +[Apr 27, 2026 - 12:55:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 127 : avg iteration time 47.44 seconds +[Apr 27, 2026 - 12:55:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 127 : data loading time 0.00 +[Apr 27, 2026 - 12:55:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 127 : avg forward pass time 11.34 +[Apr 27, 2026 - 12:55:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 127 : backward pass time 11.54 +[Apr 27, 2026 - 12:55:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 127 : optimizer step time 0.98 +[Apr 27, 2026 - 12:55:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 127-------------------- +[Apr 27, 2026 - 12:55:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0116 iter count: 1.0 +[Apr 27, 2026 - 12:55:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[Apr 27, 2026 - 12:55:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 12:55:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 12:56:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076382 43.159882 42.986542 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.992188 70.195312 69.839844 +util 97.250000 99.000000 92.000000 +[Apr 27, 2026 - 12:56:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 128 : avg iteration time 48.21 seconds +[Apr 27, 2026 - 12:56:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 128 : data loading time 0.00 +[Apr 27, 2026 - 12:56:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 128 : avg forward pass time 11.27 +[Apr 27, 2026 - 12:56:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 128 : backward pass time 11.58 +[Apr 27, 2026 - 12:56:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 128 : optimizer step time 0.97 +[Apr 27, 2026 - 12:56:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 128-------------------- +[Apr 27, 2026 - 12:56:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0111 iter count: 1.0 +[Apr 27, 2026 - 12:56:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[Apr 27, 2026 - 12:56:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 12:56:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 12:57:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076384 43.159885 42.986542 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.816406 70.058594 69.527344 +util 98.250000 100.000000 95.000000 +[Apr 27, 2026 - 12:57:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 129 : avg iteration time 47.63 seconds +[Apr 27, 2026 - 12:57:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 129 : data loading time 0.00 +[Apr 27, 2026 - 12:57:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 129 : avg forward pass time 11.27 +[Apr 27, 2026 - 12:57:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 129 : backward pass time 11.53 +[Apr 27, 2026 - 12:57:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 129 : optimizer step time 0.98 +[Apr 27, 2026 - 12:57:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 129-------------------- +[Apr 27, 2026 - 12:57:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0119 iter count: 1.0 +[Apr 27, 2026 - 12:57:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[Apr 27, 2026 - 12:57:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 12:57:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 13:00:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076275 43.159904 42.986542 +peak_gpu_mem_gb 131.581049 131.620712 131.567828 +peak_gpu_mem_reserved_gb 135.513672 135.619141 135.478516 +util 72.750000 77.000000 67.000000 +[Apr 27, 2026 - 13:00:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 130 : avg iteration time 174.02 seconds +[Apr 27, 2026 - 13:00:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 130 : data loading time 0.00 +[Apr 27, 2026 - 13:00:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 130 : avg forward pass time 78.44 +[Apr 27, 2026 - 13:00:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 130 : backward pass time 8.43 +[Apr 27, 2026 - 13:00:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 130 : optimizer step time 0.34 +[Apr 27, 2026 - 13:00:06 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 130-------------------- +[Apr 27, 2026 - 13:00:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 449063.7285 iter count: 1.0 +[Apr 27, 2026 - 13:00:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7208 iter count: 1.0 +[Apr 27, 2026 - 13:00:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 449063.7285iter count: 1.0 +[Apr 27, 2026 - 13:00:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:00:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.3202iter count: 1.0 +[Apr 27, 2026 - 13:00:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.1695 iter count: 1.0 +[Apr 27, 2026 - 13:00:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.9727 iter count: 1.0 +[Apr 27, 2026 - 13:00:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 551307.6250iter count: 1.0 +[Apr 27, 2026 - 13:00:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 10.3699iter count: 1.0 +[Apr 27, 2026 - 13:00:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 8414476.0000iter count: 1.0 +[Apr 27, 2026 - 13:00:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7208iter count: 1.0 +[Apr 27, 2026 - 13:00:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.1656 iter count: 1.0 +[Apr 27, 2026 - 13:00:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 30673.6016iter count: 1.0 +[Apr 27, 2026 - 13:00:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.6068 iter count: 1.0 +[Apr 27, 2026 - 13:00:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 74129.4141iter count: 1.0 +[Apr 27, 2026 - 13:00:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 1.6545 iter count: 1.0 +[Apr 27, 2026 - 13:00:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 27.3603 iter count: 1.0 +[Apr 27, 2026 - 13:00:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.0188 iter count: 1.0 +[Apr 27, 2026 - 13:00:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 168983.7031iter count: 1.0 +[Apr 27, 2026 - 13:00:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0127 iter count: 1.0 +[Apr 27, 2026 - 13:00:06 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:00:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 13:01:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076320 43.159882 42.986542 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.855469 70.117188 69.390625 +util 94.250000 99.000000 91.000000 +[Apr 27, 2026 - 13:01:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 131 : avg iteration time 53.87 seconds +[Apr 27, 2026 - 13:01:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 131 : data loading time 0.00 +[Apr 27, 2026 - 13:01:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 131 : avg forward pass time 14.73 +[Apr 27, 2026 - 13:01:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 131 : backward pass time 11.59 +[Apr 27, 2026 - 13:01:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 131 : optimizer step time 0.98 +[Apr 27, 2026 - 13:01:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 131-------------------- +[Apr 27, 2026 - 13:01:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0149 iter count: 1.0 +[Apr 27, 2026 - 13:01:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0149 iter count: 1.0 +[Apr 27, 2026 - 13:01:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:01:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 13:01:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076440 43.159904 42.986557 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.816406 70.039062 69.546875 +util 97.500000 100.000000 92.000000 +[Apr 27, 2026 - 13:01:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 132 : avg iteration time 50.68 seconds +[Apr 27, 2026 - 13:01:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 132 : data loading time 0.00 +[Apr 27, 2026 - 13:01:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 132 : avg forward pass time 12.92 +[Apr 27, 2026 - 13:01:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 132 : backward pass time 11.63 +[Apr 27, 2026 - 13:01:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 132 : optimizer step time 1.41 +[Apr 27, 2026 - 13:01:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 132-------------------- +[Apr 27, 2026 - 13:01:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0125 iter count: 1.0 +[Apr 27, 2026 - 13:01:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0125 iter count: 1.0 +[Apr 27, 2026 - 13:01:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:01:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 13:02:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076491 43.159882 42.986542 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.918945 70.000000 69.859375 +util 96.750000 99.000000 91.000000 +[Apr 27, 2026 - 13:02:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 133 : avg iteration time 50.67 seconds +[Apr 27, 2026 - 13:02:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 133 : data loading time 0.00 +[Apr 27, 2026 - 13:02:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 133 : avg forward pass time 12.59 +[Apr 27, 2026 - 13:02:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 133 : backward pass time 12.85 +[Apr 27, 2026 - 13:02:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 133 : optimizer step time 1.07 +[Apr 27, 2026 - 13:02:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 133-------------------- +[Apr 27, 2026 - 13:02:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0138 iter count: 1.0 +[Apr 27, 2026 - 13:02:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0138 iter count: 1.0 +[Apr 27, 2026 - 13:02:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:02:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 13:03:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076491 43.159882 42.986542 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.840820 70.039062 69.527344 +util 97.500000 99.000000 93.000000 +[Apr 27, 2026 - 13:03:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 134 : avg iteration time 51.83 seconds +[Apr 27, 2026 - 13:03:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 134 : data loading time 0.00 +[Apr 27, 2026 - 13:03:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 134 : avg forward pass time 13.88 +[Apr 27, 2026 - 13:03:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 134 : backward pass time 11.56 +[Apr 27, 2026 - 13:03:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 134 : optimizer step time 0.97 +[Apr 27, 2026 - 13:03:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 134-------------------- +[Apr 27, 2026 - 13:03:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0142 iter count: 1.0 +[Apr 27, 2026 - 13:03:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0142 iter count: 1.0 +[Apr 27, 2026 - 13:03:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:03:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 13:06:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076171 43.159882 42.986542 +peak_gpu_mem_gb 131.581049 131.620712 131.567828 +peak_gpu_mem_reserved_gb 135.547852 135.697266 135.458984 +util 69.000000 74.000000 63.000000 +[Apr 27, 2026 - 13:06:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 135 : avg iteration time 172.92 seconds +[Apr 27, 2026 - 13:06:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 135 : data loading time 0.00 +[Apr 27, 2026 - 13:06:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 135 : avg forward pass time 77.94 +[Apr 27, 2026 - 13:06:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 135 : backward pass time 8.39 +[Apr 27, 2026 - 13:06:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 135 : optimizer step time 0.34 +[Apr 27, 2026 - 13:06:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 135-------------------- +[Apr 27, 2026 - 13:06:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 108423.7714 iter count: 1.0 +[Apr 27, 2026 - 13:06:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7556 iter count: 1.0 +[Apr 27, 2026 - 13:06:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 108423.7714iter count: 1.0 +[Apr 27, 2026 - 13:06:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:06:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 3.8663iter count: 1.0 +[Apr 27, 2026 - 13:06:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.8284 iter count: 1.0 +[Apr 27, 2026 - 13:06:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.6198 iter count: 1.0 +[Apr 27, 2026 - 13:06:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 173618.4062iter count: 1.0 +[Apr 27, 2026 - 13:06:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 5.2425 iter count: 1.0 +[Apr 27, 2026 - 13:06:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 4154331.7500iter count: 1.0 +[Apr 27, 2026 - 13:06:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7556iter count: 1.0 +[Apr 27, 2026 - 13:06:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.2888 iter count: 1.0 +[Apr 27, 2026 - 13:06:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 5311.8418iter count: 1.0 +[Apr 27, 2026 - 13:06:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 1.7559 iter count: 1.0 +[Apr 27, 2026 - 13:06:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 33.5071 iter count: 1.0 +[Apr 27, 2026 - 13:06:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.6133 iter count: 1.0 +[Apr 27, 2026 - 13:06:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 1375.6672iter count: 1.0 +[Apr 27, 2026 - 13:06:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 2.0545 iter count: 1.0 +[Apr 27, 2026 - 13:06:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 60.8869 iter count: 1.0 +[Apr 27, 2026 - 13:06:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0128 iter count: 1.0 +[Apr 27, 2026 - 13:06:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:06:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 13:07:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076180 43.159882 42.986542 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.909668 70.001953 69.839844 +util 98.750000 100.000000 95.000000 +[Apr 27, 2026 - 13:07:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 136 : avg iteration time 51.75 seconds +[Apr 27, 2026 - 13:07:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 136 : data loading time 0.00 +[Apr 27, 2026 - 13:07:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 136 : avg forward pass time 13.46 +[Apr 27, 2026 - 13:07:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 136 : backward pass time 11.59 +[Apr 27, 2026 - 13:07:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 136 : optimizer step time 1.34 +[Apr 27, 2026 - 13:07:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 136-------------------- +[Apr 27, 2026 - 13:07:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0124 iter count: 1.0 +[Apr 27, 2026 - 13:07:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0124 iter count: 1.0 +[Apr 27, 2026 - 13:07:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:07:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 13:08:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076707 43.159882 42.986542 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.973145 70.015625 69.878906 +util 97.250000 99.000000 92.000000 +[Apr 27, 2026 - 13:08:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 137 : avg iteration time 50.53 seconds +[Apr 27, 2026 - 13:08:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 137 : data loading time 0.00 +[Apr 27, 2026 - 13:08:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 137 : avg forward pass time 12.58 +[Apr 27, 2026 - 13:08:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 137 : backward pass time 12.75 +[Apr 27, 2026 - 13:08:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 137 : optimizer step time 1.05 +[Apr 27, 2026 - 13:08:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 137-------------------- +[Apr 27, 2026 - 13:08:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0145 iter count: 1.0 +[Apr 27, 2026 - 13:08:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0145 iter count: 1.0 +[Apr 27, 2026 - 13:08:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:08:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 13:08:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076715 43.159885 42.986542 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.982910 70.080078 69.898438 +util 97.750000 100.000000 93.000000 +[Apr 27, 2026 - 13:08:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 138 : avg iteration time 48.93 seconds +[Apr 27, 2026 - 13:08:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 138 : data loading time 0.00 +[Apr 27, 2026 - 13:08:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 138 : avg forward pass time 12.43 +[Apr 27, 2026 - 13:08:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 138 : backward pass time 11.55 +[Apr 27, 2026 - 13:08:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 138 : optimizer step time 0.98 +[Apr 27, 2026 - 13:08:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 138-------------------- +[Apr 27, 2026 - 13:08:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0120 iter count: 1.0 +[Apr 27, 2026 - 13:08:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0120 iter count: 1.0 +[Apr 27, 2026 - 13:08:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:08:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 13:09:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076707 43.159882 42.986542 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.992676 70.099609 69.898438 +util 96.750000 99.000000 94.000000 +[Apr 27, 2026 - 13:09:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 139 : avg iteration time 50.53 seconds +[Apr 27, 2026 - 13:09:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 139 : data loading time 0.00 +[Apr 27, 2026 - 13:09:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 139 : avg forward pass time 12.50 +[Apr 27, 2026 - 13:09:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 139 : backward pass time 11.57 +[Apr 27, 2026 - 13:09:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 139 : optimizer step time 0.98 +[Apr 27, 2026 - 13:09:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 139-------------------- +[Apr 27, 2026 - 13:09:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0132 iter count: 1.0 +[Apr 27, 2026 - 13:09:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0132 iter count: 1.0 +[Apr 27, 2026 - 13:09:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:09:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 13:12:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076380 43.159882 42.986542 +peak_gpu_mem_gb 131.581049 131.620712 131.567828 +peak_gpu_mem_reserved_gb 135.561523 135.677734 135.515625 +util 67.500000 78.000000 57.000000 +[Apr 27, 2026 - 13:12:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 140 : avg iteration time 177.74 seconds +[Apr 27, 2026 - 13:12:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 140 : data loading time 0.00 +[Apr 27, 2026 - 13:12:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 140 : avg forward pass time 79.44 +[Apr 27, 2026 - 13:12:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 140 : backward pass time 8.89 +[Apr 27, 2026 - 13:12:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 140 : optimizer step time 0.34 +[Apr 27, 2026 - 13:12:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 140-------------------- +[Apr 27, 2026 - 13:12:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 45238.1191 iter count: 1.0 +[Apr 27, 2026 - 13:12:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6739 iter count: 1.0 +[Apr 27, 2026 - 13:12:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 45238.1191iter count: 1.0 +[Apr 27, 2026 - 13:12:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:12:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.1903iter count: 1.0 +[Apr 27, 2026 - 13:12:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.8087 iter count: 1.0 +[Apr 27, 2026 - 13:12:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 6.6707 iter count: 1.0 +[Apr 27, 2026 - 13:12:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 71665.1562iter count: 1.0 +[Apr 27, 2026 - 13:12:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 5.0403 iter count: 1.0 +[Apr 27, 2026 - 13:12:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 622461.0625iter count: 1.0 +[Apr 27, 2026 - 13:12:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6739iter count: 1.0 +[Apr 27, 2026 - 13:12:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.3078 iter count: 1.0 +[Apr 27, 2026 - 13:12:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 40763.3867iter count: 1.0 +[Apr 27, 2026 - 13:12:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.2636 iter count: 1.0 +[Apr 27, 2026 - 13:12:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 37319.4648iter count: 1.0 +[Apr 27, 2026 - 13:12:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.8629 iter count: 1.0 +[Apr 27, 2026 - 13:12:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 123735.1641iter count: 1.0 +[Apr 27, 2026 - 13:12:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.8275 iter count: 1.0 +[Apr 27, 2026 - 13:12:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 2111.2693iter count: 1.0 +[Apr 27, 2026 - 13:12:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[Apr 27, 2026 - 13:12:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:12:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 13:13:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076385 43.159882 42.986542 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.918945 70.117188 69.839844 +util 98.000000 99.000000 97.000000 +[Apr 27, 2026 - 13:13:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 141 : avg iteration time 50.91 seconds +[Apr 27, 2026 - 13:13:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 141 : data loading time 0.00 +[Apr 27, 2026 - 13:13:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 141 : avg forward pass time 13.23 +[Apr 27, 2026 - 13:13:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 141 : backward pass time 11.57 +[Apr 27, 2026 - 13:13:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 141 : optimizer step time 0.97 +[Apr 27, 2026 - 13:13:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 141-------------------- +[Apr 27, 2026 - 13:13:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0121 iter count: 1.0 +[Apr 27, 2026 - 13:13:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[Apr 27, 2026 - 13:13:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:13:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 13:14:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076753 43.159897 42.986542 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.889648 70.039062 69.820312 +util 98.250000 100.000000 95.000000 +[Apr 27, 2026 - 13:14:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 142 : avg iteration time 50.65 seconds +[Apr 27, 2026 - 13:14:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 142 : data loading time 0.00 +[Apr 27, 2026 - 13:14:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 142 : avg forward pass time 12.56 +[Apr 27, 2026 - 13:14:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 142 : backward pass time 11.53 +[Apr 27, 2026 - 13:14:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 142 : optimizer step time 0.99 +[Apr 27, 2026 - 13:14:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 142-------------------- +[Apr 27, 2026 - 13:14:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0132 iter count: 1.0 +[Apr 27, 2026 - 13:14:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0132 iter count: 1.0 +[Apr 27, 2026 - 13:14:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:14:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 13:15:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076809 43.159897 42.986542 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.943359 70.253906 69.820312 +util 98.250000 100.000000 94.000000 +[Apr 27, 2026 - 13:15:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 143 : avg iteration time 51.66 seconds +[Apr 27, 2026 - 13:15:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 143 : data loading time 0.00 +[Apr 27, 2026 - 13:15:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 143 : avg forward pass time 12.97 +[Apr 27, 2026 - 13:15:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 143 : backward pass time 12.79 +[Apr 27, 2026 - 13:15:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 143 : optimizer step time 1.40 +[Apr 27, 2026 - 13:15:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 143-------------------- +[Apr 27, 2026 - 13:15:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0121 iter count: 1.0 +[Apr 27, 2026 - 13:15:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[Apr 27, 2026 - 13:15:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:15:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 13:16:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076724 43.159893 42.986542 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.953125 70.058594 69.878906 +util 96.500000 99.000000 91.000000 +[Apr 27, 2026 - 13:16:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 144 : avg iteration time 48.16 seconds +[Apr 27, 2026 - 13:16:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 144 : data loading time 0.00 +[Apr 27, 2026 - 13:16:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 144 : avg forward pass time 11.39 +[Apr 27, 2026 - 13:16:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 144 : backward pass time 12.74 +[Apr 27, 2026 - 13:16:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 144 : optimizer step time 1.04 +[Apr 27, 2026 - 13:16:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 144-------------------- +[Apr 27, 2026 - 13:16:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0119 iter count: 1.0 +[Apr 27, 2026 - 13:16:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[Apr 27, 2026 - 13:16:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:16:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 13:19:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076518 43.159904 42.986542 +peak_gpu_mem_gb 131.581049 131.620712 131.567828 +peak_gpu_mem_reserved_gb 135.577148 135.619141 135.498047 +util 71.250000 78.000000 67.000000 +[Apr 27, 2026 - 13:19:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 145 : avg iteration time 174.52 seconds +[Apr 27, 2026 - 13:19:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 145 : data loading time 0.00 +[Apr 27, 2026 - 13:19:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 145 : avg forward pass time 78.74 +[Apr 27, 2026 - 13:19:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 145 : backward pass time 8.42 +[Apr 27, 2026 - 13:19:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 145 : optimizer step time 0.33 +[Apr 27, 2026 - 13:19:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 145-------------------- +[Apr 27, 2026 - 13:19:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 35438.6279 iter count: 1.0 +[Apr 27, 2026 - 13:19:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7088 iter count: 1.0 +[Apr 27, 2026 - 13:19:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 35438.6279iter count: 1.0 +[Apr 27, 2026 - 13:19:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:19:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.5758iter count: 1.0 +[Apr 27, 2026 - 13:19:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 2.1714 iter count: 1.0 +[Apr 27, 2026 - 13:19:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 6.2321 iter count: 1.0 +[Apr 27, 2026 - 13:19:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 47139.6953iter count: 1.0 +[Apr 27, 2026 - 13:19:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 76.9229iter count: 1.0 +[Apr 27, 2026 - 13:19:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 258876.2969iter count: 1.0 +[Apr 27, 2026 - 13:19:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7088iter count: 1.0 +[Apr 27, 2026 - 13:19:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.8141 iter count: 1.0 +[Apr 27, 2026 - 13:19:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 112215.6406iter count: 1.0 +[Apr 27, 2026 - 13:19:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 2.1714 iter count: 1.0 +[Apr 27, 2026 - 13:19:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 76.9229 iter count: 1.0 +[Apr 27, 2026 - 13:19:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.8549 iter count: 1.0 +[Apr 27, 2026 - 13:19:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 16478.6016iter count: 1.0 +[Apr 27, 2026 - 13:19:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.5771 iter count: 1.0 +[Apr 27, 2026 - 13:19:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 1279.4313iter count: 1.0 +[Apr 27, 2026 - 13:19:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[Apr 27, 2026 - 13:19:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:19:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 13:19:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076506 43.159882 42.986542 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.475586 72.636719 69.703125 +util 98.000000 99.000000 97.000000 +[Apr 27, 2026 - 13:19:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 146 : avg iteration time 52.19 seconds +[Apr 27, 2026 - 13:19:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 146 : data loading time 0.00 +[Apr 27, 2026 - 13:19:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 146 : avg forward pass time 13.33 +[Apr 27, 2026 - 13:19:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 146 : backward pass time 11.54 +[Apr 27, 2026 - 13:19:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 146 : optimizer step time 0.97 +[Apr 27, 2026 - 13:19:54 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 146-------------------- +[Apr 27, 2026 - 13:19:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0152 iter count: 1.0 +[Apr 27, 2026 - 13:19:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0152 iter count: 1.0 +[Apr 27, 2026 - 13:19:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:19:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 13:20:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076833 43.159882 42.986542 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.694336 69.839844 69.566406 +util 97.750000 100.000000 93.000000 +[Apr 27, 2026 - 13:20:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 147 : avg iteration time 54.34 seconds +[Apr 27, 2026 - 13:20:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 147 : data loading time 0.00 +[Apr 27, 2026 - 13:20:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 147 : avg forward pass time 14.27 +[Apr 27, 2026 - 13:20:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 147 : backward pass time 12.88 +[Apr 27, 2026 - 13:20:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 147 : optimizer step time 1.34 +[Apr 27, 2026 - 13:20:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 147-------------------- +[Apr 27, 2026 - 13:20:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0135 iter count: 1.0 +[Apr 27, 2026 - 13:20:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0135 iter count: 1.0 +[Apr 27, 2026 - 13:20:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:20:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 13:21:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076844 43.159882 42.986553 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.674805 69.878906 69.507812 +util 98.000000 100.000000 96.000000 +[Apr 27, 2026 - 13:21:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 148 : avg iteration time 51.27 seconds +[Apr 27, 2026 - 13:21:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 148 : data loading time 0.00 +[Apr 27, 2026 - 13:21:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 148 : avg forward pass time 12.84 +[Apr 27, 2026 - 13:21:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 148 : backward pass time 12.82 +[Apr 27, 2026 - 13:21:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 148 : optimizer step time 1.25 +[Apr 27, 2026 - 13:21:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 148-------------------- +[Apr 27, 2026 - 13:21:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0108 iter count: 1.0 +[Apr 27, 2026 - 13:21:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[Apr 27, 2026 - 13:21:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:21:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 13:22:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076839 43.159882 42.986542 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.016602 70.093750 69.804688 +util 96.750000 99.000000 92.000000 +[Apr 27, 2026 - 13:22:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 149 : avg iteration time 47.48 seconds +[Apr 27, 2026 - 13:22:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 149 : data loading time 0.00 +[Apr 27, 2026 - 13:22:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 149 : avg forward pass time 11.35 +[Apr 27, 2026 - 13:22:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 149 : backward pass time 12.22 +[Apr 27, 2026 - 13:22:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 149 : optimizer step time 1.00 +[Apr 27, 2026 - 13:22:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 149-------------------- +[Apr 27, 2026 - 13:22:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0117 iter count: 1.0 +[Apr 27, 2026 - 13:22:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[Apr 27, 2026 - 13:22:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:22:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 13:25:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076528 43.159882 42.986549 +peak_gpu_mem_gb 131.581049 131.620712 131.567828 +peak_gpu_mem_reserved_gb 135.528320 135.619141 135.458984 +util 71.750000 77.000000 64.000000 +[Apr 27, 2026 - 13:25:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 150 : avg iteration time 173.42 seconds +[Apr 27, 2026 - 13:25:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 150 : data loading time 0.00 +[Apr 27, 2026 - 13:25:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 150 : avg forward pass time 78.16 +[Apr 27, 2026 - 13:25:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 150 : backward pass time 8.42 +[Apr 27, 2026 - 13:25:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 150 : optimizer step time 0.34 +[Apr 27, 2026 - 13:25:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 150-------------------- +[Apr 27, 2026 - 13:25:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 109505.4365 iter count: 1.0 +[Apr 27, 2026 - 13:25:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7084 iter count: 1.0 +[Apr 27, 2026 - 13:25:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 109505.4365iter count: 1.0 +[Apr 27, 2026 - 13:25:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:25:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.5511iter count: 1.0 +[Apr 27, 2026 - 13:25:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.1466 iter count: 1.0 +[Apr 27, 2026 - 13:25:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.3537 iter count: 1.0 +[Apr 27, 2026 - 13:25:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 166264.2812iter count: 1.0 +[Apr 27, 2026 - 13:25:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 9.9074 iter count: 1.0 +[Apr 27, 2026 - 13:25:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 2439938.0000iter count: 1.0 +[Apr 27, 2026 - 13:25:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7084iter count: 1.0 +[Apr 27, 2026 - 13:25:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 2.6020 iter count: 1.0 +[Apr 27, 2026 - 13:25:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 182.0072iter count: 1.0 +[Apr 27, 2026 - 13:25:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.7286 iter count: 1.0 +[Apr 27, 2026 - 13:25:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 94575.1016iter count: 1.0 +[Apr 27, 2026 - 13:25:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 6.5948 iter count: 1.0 +[Apr 27, 2026 - 13:25:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 534765.0625iter count: 1.0 +[Apr 27, 2026 - 13:25:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.4216 iter count: 1.0 +[Apr 27, 2026 - 13:25:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 6927.3267iter count: 1.0 +[Apr 27, 2026 - 13:25:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0125 iter count: 1.0 +[Apr 27, 2026 - 13:25:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:25:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 13:26:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076531 43.159882 42.986565 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.758301 69.962891 69.644531 +util 97.250000 100.000000 91.000000 +[Apr 27, 2026 - 13:26:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 151 : avg iteration time 51.70 seconds +[Apr 27, 2026 - 13:26:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 151 : data loading time 0.00 +[Apr 27, 2026 - 13:26:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 151 : avg forward pass time 13.45 +[Apr 27, 2026 - 13:26:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 151 : backward pass time 11.57 +[Apr 27, 2026 - 13:26:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 151 : optimizer step time 1.33 +[Apr 27, 2026 - 13:26:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 151-------------------- +[Apr 27, 2026 - 13:26:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0126 iter count: 1.0 +[Apr 27, 2026 - 13:26:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0126 iter count: 1.0 +[Apr 27, 2026 - 13:26:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:26:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 13:27:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076536 43.159882 42.986542 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.802246 69.982422 69.625000 +util 97.500000 99.000000 93.000000 +[Apr 27, 2026 - 13:27:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 152 : avg iteration time 52.89 seconds +[Apr 27, 2026 - 13:27:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 152 : data loading time 0.00 +[Apr 27, 2026 - 13:27:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 152 : avg forward pass time 14.36 +[Apr 27, 2026 - 13:27:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 152 : backward pass time 11.59 +[Apr 27, 2026 - 13:27:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 152 : optimizer step time 0.97 +[Apr 27, 2026 - 13:27:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 152-------------------- +[Apr 27, 2026 - 13:27:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0136 iter count: 1.0 +[Apr 27, 2026 - 13:27:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0136 iter count: 1.0 +[Apr 27, 2026 - 13:27:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:27:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 13:27:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076545 43.159904 42.986565 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.792480 69.982422 69.625000 +util 97.500000 100.000000 92.000000 +[Apr 27, 2026 - 13:27:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 153 : avg iteration time 50.32 seconds +[Apr 27, 2026 - 13:27:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 153 : data loading time 0.00 +[Apr 27, 2026 - 13:27:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 153 : avg forward pass time 12.83 +[Apr 27, 2026 - 13:27:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 153 : backward pass time 11.56 +[Apr 27, 2026 - 13:27:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 153 : optimizer step time 0.98 +[Apr 27, 2026 - 13:27:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 153-------------------- +[Apr 27, 2026 - 13:27:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0118 iter count: 1.0 +[Apr 27, 2026 - 13:27:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0118 iter count: 1.0 +[Apr 27, 2026 - 13:27:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:27:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 13:28:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076829 43.159889 42.986565 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.807129 70.080078 69.625000 +util 98.000000 100.000000 94.000000 +[Apr 27, 2026 - 13:28:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 154 : avg iteration time 47.75 seconds +[Apr 27, 2026 - 13:28:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 154 : data loading time 0.00 +[Apr 27, 2026 - 13:28:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 154 : avg forward pass time 11.39 +[Apr 27, 2026 - 13:28:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 154 : backward pass time 11.56 +[Apr 27, 2026 - 13:28:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 154 : optimizer step time 1.18 +[Apr 27, 2026 - 13:28:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 154-------------------- +[Apr 27, 2026 - 13:28:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0124 iter count: 1.0 +[Apr 27, 2026 - 13:28:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0124 iter count: 1.0 +[Apr 27, 2026 - 13:28:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:28:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 13:31:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076589 43.159885 42.986542 +peak_gpu_mem_gb 131.581049 131.620712 131.567828 +peak_gpu_mem_reserved_gb 135.581055 135.677734 135.498047 +util 68.750000 73.000000 63.000000 +[Apr 27, 2026 - 13:31:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 155 : avg iteration time 170.46 seconds +[Apr 27, 2026 - 13:31:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 155 : data loading time 0.00 +[Apr 27, 2026 - 13:31:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 155 : avg forward pass time 76.26 +[Apr 27, 2026 - 13:31:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 155 : backward pass time 8.33 +[Apr 27, 2026 - 13:31:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 155 : optimizer step time 0.34 +[Apr 27, 2026 - 13:31:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 155-------------------- +[Apr 27, 2026 - 13:31:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 644720.3760 iter count: 1.0 +[Apr 27, 2026 - 13:31:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6324 iter count: 1.0 +[Apr 27, 2026 - 13:31:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 644720.3760iter count: 1.0 +[Apr 27, 2026 - 13:31:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:31:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.6969iter count: 1.0 +[Apr 27, 2026 - 13:31:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.9813 iter count: 1.0 +[Apr 27, 2026 - 13:31:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.5847 iter count: 1.0 +[Apr 27, 2026 - 13:31:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 1188444.5000iter count: 1.0 +[Apr 27, 2026 - 13:31:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 52.5983iter count: 1.0 +[Apr 27, 2026 - 13:31:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 28611332.0000iter count: 1.0 +[Apr 27, 2026 - 13:31:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6324iter count: 1.0 +[Apr 27, 2026 - 13:31:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.4779 iter count: 1.0 +[Apr 27, 2026 - 13:31:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 7753.3418iter count: 1.0 +[Apr 27, 2026 - 13:31:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.1882 iter count: 1.0 +[Apr 27, 2026 - 13:31:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 32090.6465iter count: 1.0 +[Apr 27, 2026 - 13:31:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.4220 iter count: 1.0 +[Apr 27, 2026 - 13:31:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 938.2899iter count: 1.0 +[Apr 27, 2026 - 13:31:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.7155 iter count: 1.0 +[Apr 27, 2026 - 13:31:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 680844.0000iter count: 1.0 +[Apr 27, 2026 - 13:31:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[Apr 27, 2026 - 13:31:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:31:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 13:32:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076593 43.159904 42.986565 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.816406 70.054688 69.664062 +util 97.750000 99.000000 94.000000 +[Apr 27, 2026 - 13:32:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 156 : avg iteration time 54.42 seconds +[Apr 27, 2026 - 13:32:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 156 : data loading time 0.00 +[Apr 27, 2026 - 13:32:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 156 : avg forward pass time 14.80 +[Apr 27, 2026 - 13:32:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 156 : backward pass time 11.53 +[Apr 27, 2026 - 13:32:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 156 : optimizer step time 0.98 +[Apr 27, 2026 - 13:32:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 156-------------------- +[Apr 27, 2026 - 13:32:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0132 iter count: 1.0 +[Apr 27, 2026 - 13:32:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0132 iter count: 1.0 +[Apr 27, 2026 - 13:32:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:32:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 13:33:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076582 43.159882 42.986542 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.860352 70.195312 69.664062 +util 97.500000 99.000000 93.000000 +[Apr 27, 2026 - 13:33:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 157 : avg iteration time 50.58 seconds +[Apr 27, 2026 - 13:33:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 157 : data loading time 0.00 +[Apr 27, 2026 - 13:33:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 157 : avg forward pass time 12.63 +[Apr 27, 2026 - 13:33:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 157 : backward pass time 11.60 +[Apr 27, 2026 - 13:33:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 157 : optimizer step time 1.41 +[Apr 27, 2026 - 13:33:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 157-------------------- +[Apr 27, 2026 - 13:33:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0125 iter count: 1.0 +[Apr 27, 2026 - 13:33:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0125 iter count: 1.0 +[Apr 27, 2026 - 13:33:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:33:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 13:34:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076593 43.159904 42.986561 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.875000 70.195312 69.664062 +util 97.500000 99.000000 93.000000 +[Apr 27, 2026 - 13:34:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 158 : avg iteration time 50.21 seconds +[Apr 27, 2026 - 13:34:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 158 : data loading time 0.00 +[Apr 27, 2026 - 13:34:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 158 : avg forward pass time 12.61 +[Apr 27, 2026 - 13:34:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 158 : backward pass time 12.38 +[Apr 27, 2026 - 13:34:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 158 : optimizer step time 1.01 +[Apr 27, 2026 - 13:34:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 158-------------------- +[Apr 27, 2026 - 13:34:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0125 iter count: 1.0 +[Apr 27, 2026 - 13:34:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0125 iter count: 1.0 +[Apr 27, 2026 - 13:34:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:34:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 13:34:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076590 43.159885 42.986542 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.021484 70.097656 69.878906 +util 97.750000 99.000000 94.000000 +[Apr 27, 2026 - 13:34:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 159 : avg iteration time 49.41 seconds +[Apr 27, 2026 - 13:34:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 159 : data loading time 0.00 +[Apr 27, 2026 - 13:34:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 159 : avg forward pass time 12.66 +[Apr 27, 2026 - 13:34:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 159 : backward pass time 11.53 +[Apr 27, 2026 - 13:34:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 159 : optimizer step time 0.97 +[Apr 27, 2026 - 13:34:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 159-------------------- +[Apr 27, 2026 - 13:34:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0119 iter count: 1.0 +[Apr 27, 2026 - 13:34:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[Apr 27, 2026 - 13:34:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:34:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 13:37:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076650 43.159885 42.986542 +peak_gpu_mem_gb 131.581049 131.620712 131.567828 +peak_gpu_mem_reserved_gb 135.522949 135.697266 135.437500 +util 72.250000 76.000000 65.000000 +[Apr 27, 2026 - 13:37:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 160 : avg iteration time 177.59 seconds +[Apr 27, 2026 - 13:37:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 160 : data loading time 0.00 +[Apr 27, 2026 - 13:37:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 160 : avg forward pass time 79.70 +[Apr 27, 2026 - 13:37:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 160 : backward pass time 9.59 +[Apr 27, 2026 - 13:37:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 160 : optimizer step time 0.34 +[Apr 27, 2026 - 13:37:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 160-------------------- +[Apr 27, 2026 - 13:37:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 100435.2437 iter count: 1.0 +[Apr 27, 2026 - 13:37:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7328 iter count: 1.0 +[Apr 27, 2026 - 13:37:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 100435.2437iter count: 1.0 +[Apr 27, 2026 - 13:37:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:37:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.3273iter count: 1.0 +[Apr 27, 2026 - 13:37:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.6535 iter count: 1.0 +[Apr 27, 2026 - 13:37:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 6.9847 iter count: 1.0 +[Apr 27, 2026 - 13:37:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 130528.4375iter count: 1.0 +[Apr 27, 2026 - 13:37:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 3.6949 iter count: 1.0 +[Apr 27, 2026 - 13:37:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 1166398.0000iter count: 1.0 +[Apr 27, 2026 - 13:37:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7328iter count: 1.0 +[Apr 27, 2026 - 13:37:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.7580 iter count: 1.0 +[Apr 27, 2026 - 13:37:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 100308.7734iter count: 1.0 +[Apr 27, 2026 - 13:37:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.7701 iter count: 1.0 +[Apr 27, 2026 - 13:37:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 13908.8066iter count: 1.0 +[Apr 27, 2026 - 13:37:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.1707 iter count: 1.0 +[Apr 27, 2026 - 13:37:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 567.5526iter count: 1.0 +[Apr 27, 2026 - 13:37:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.5884 iter count: 1.0 +[Apr 27, 2026 - 13:37:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 1308.7458iter count: 1.0 +[Apr 27, 2026 - 13:37:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0120 iter count: 1.0 +[Apr 27, 2026 - 13:37:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:37:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 13:38:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076658 43.159908 42.986565 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.992188 70.039062 69.898438 +util 97.750000 100.000000 93.000000 +[Apr 27, 2026 - 13:38:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 161 : avg iteration time 49.06 seconds +[Apr 27, 2026 - 13:38:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 161 : data loading time 0.00 +[Apr 27, 2026 - 13:38:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 161 : avg forward pass time 12.14 +[Apr 27, 2026 - 13:38:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 161 : backward pass time 12.21 +[Apr 27, 2026 - 13:38:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 161 : optimizer step time 1.00 +[Apr 27, 2026 - 13:38:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 161-------------------- +[Apr 27, 2026 - 13:38:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0118 iter count: 1.0 +[Apr 27, 2026 - 13:38:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0118 iter count: 1.0 +[Apr 27, 2026 - 13:38:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:38:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 13:39:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076644 43.159885 42.986542 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.982422 70.039062 69.820312 +util 96.750000 100.000000 94.000000 +[Apr 27, 2026 - 13:39:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 162 : avg iteration time 49.07 seconds +[Apr 27, 2026 - 13:39:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 162 : data loading time 0.00 +[Apr 27, 2026 - 13:39:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 162 : avg forward pass time 12.49 +[Apr 27, 2026 - 13:39:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 162 : backward pass time 11.55 +[Apr 27, 2026 - 13:39:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 162 : optimizer step time 0.98 +[Apr 27, 2026 - 13:39:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 162-------------------- +[Apr 27, 2026 - 13:39:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0123 iter count: 1.0 +[Apr 27, 2026 - 13:39:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0123 iter count: 1.0 +[Apr 27, 2026 - 13:39:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:39:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 13:40:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076650 43.159908 42.986542 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.006836 70.039062 69.976562 +util 98.500000 100.000000 95.000000 +[Apr 27, 2026 - 13:40:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 163 : avg iteration time 50.60 seconds +[Apr 27, 2026 - 13:40:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 163 : data loading time 0.00 +[Apr 27, 2026 - 13:40:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 163 : avg forward pass time 12.60 +[Apr 27, 2026 - 13:40:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 163 : backward pass time 11.69 +[Apr 27, 2026 - 13:40:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 163 : optimizer step time 1.47 +[Apr 27, 2026 - 13:40:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 163-------------------- +[Apr 27, 2026 - 13:40:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0133 iter count: 1.0 +[Apr 27, 2026 - 13:40:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0133 iter count: 1.0 +[Apr 27, 2026 - 13:40:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:40:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 13:41:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076653 43.159916 42.986542 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.967773 70.039062 69.878906 +util 98.000000 99.000000 95.000000 +[Apr 27, 2026 - 13:41:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 164 : avg iteration time 50.07 seconds +[Apr 27, 2026 - 13:41:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 164 : data loading time 0.00 +[Apr 27, 2026 - 13:41:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 164 : avg forward pass time 12.69 +[Apr 27, 2026 - 13:41:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 164 : backward pass time 12.15 +[Apr 27, 2026 - 13:41:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 164 : optimizer step time 1.00 +[Apr 27, 2026 - 13:41:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 164-------------------- +[Apr 27, 2026 - 13:41:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0119 iter count: 1.0 +[Apr 27, 2026 - 13:41:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[Apr 27, 2026 - 13:41:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:41:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 13:44:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076425 43.159908 42.986542 +peak_gpu_mem_gb 131.581049 131.620712 131.567828 +peak_gpu_mem_reserved_gb 135.611328 135.775391 135.517578 +util 70.000000 75.000000 62.000000 +[Apr 27, 2026 - 13:44:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 165 : avg iteration time 173.84 seconds +[Apr 27, 2026 - 13:44:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 165 : data loading time 0.00 +[Apr 27, 2026 - 13:44:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 165 : avg forward pass time 78.37 +[Apr 27, 2026 - 13:44:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 165 : backward pass time 8.40 +[Apr 27, 2026 - 13:44:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 165 : optimizer step time 0.34 +[Apr 27, 2026 - 13:44:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 165-------------------- +[Apr 27, 2026 - 13:44:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 170871.5337 iter count: 1.0 +[Apr 27, 2026 - 13:44:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7133 iter count: 1.0 +[Apr 27, 2026 - 13:44:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 170871.5337iter count: 1.0 +[Apr 27, 2026 - 13:44:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:44:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.8822iter count: 1.0 +[Apr 27, 2026 - 13:44:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 2.3759 iter count: 1.0 +[Apr 27, 2026 - 13:44:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.6379 iter count: 1.0 +[Apr 27, 2026 - 13:44:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 205938.1406iter count: 1.0 +[Apr 27, 2026 - 13:44:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 115.8021iter count: 1.0 +[Apr 27, 2026 - 13:44:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 4307044.5000iter count: 1.0 +[Apr 27, 2026 - 13:44:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7133iter count: 1.0 +[Apr 27, 2026 - 13:44:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.3477 iter count: 1.0 +[Apr 27, 2026 - 13:44:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 44152.9727iter count: 1.0 +[Apr 27, 2026 - 13:44:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.5554 iter count: 1.0 +[Apr 27, 2026 - 13:44:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 66884.8828iter count: 1.0 +[Apr 27, 2026 - 13:44:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.0443 iter count: 1.0 +[Apr 27, 2026 - 13:44:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 440.7888iter count: 1.0 +[Apr 27, 2026 - 13:44:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 2.7008 iter count: 1.0 +[Apr 27, 2026 - 13:44:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 221.7652iter count: 1.0 +[Apr 27, 2026 - 13:44:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0124 iter count: 1.0 +[Apr 27, 2026 - 13:44:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:44:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 13:45:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076428 43.159931 42.986542 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.943848 70.158203 69.859375 +util 96.750000 100.000000 92.000000 +[Apr 27, 2026 - 13:45:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 166 : avg iteration time 55.04 seconds +[Apr 27, 2026 - 13:45:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 166 : data loading time 0.00 +[Apr 27, 2026 - 13:45:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 166 : avg forward pass time 14.79 +[Apr 27, 2026 - 13:45:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 166 : backward pass time 12.17 +[Apr 27, 2026 - 13:45:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 166 : optimizer step time 1.56 +[Apr 27, 2026 - 13:45:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 166-------------------- +[Apr 27, 2026 - 13:45:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0150 iter count: 1.0 +[Apr 27, 2026 - 13:45:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0150 iter count: 1.0 +[Apr 27, 2026 - 13:45:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:45:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 13:45:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076777 43.159885 42.986542 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.918945 70.175781 69.820312 +util 95.000000 99.000000 92.000000 +[Apr 27, 2026 - 13:45:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 167 : avg iteration time 52.62 seconds +[Apr 27, 2026 - 13:45:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 167 : data loading time 0.00 +[Apr 27, 2026 - 13:45:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 167 : avg forward pass time 14.23 +[Apr 27, 2026 - 13:45:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 167 : backward pass time 11.60 +[Apr 27, 2026 - 13:45:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 167 : optimizer step time 0.98 +[Apr 27, 2026 - 13:45:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 167-------------------- +[Apr 27, 2026 - 13:45:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0140 iter count: 1.0 +[Apr 27, 2026 - 13:45:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0140 iter count: 1.0 +[Apr 27, 2026 - 13:45:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:45:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 13:46:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076982 43.159885 42.986549 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.938477 70.175781 69.839844 +util 96.000000 99.000000 90.000000 +[Apr 27, 2026 - 13:46:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 168 : avg iteration time 50.52 seconds +[Apr 27, 2026 - 13:46:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 168 : data loading time 0.00 +[Apr 27, 2026 - 13:46:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 168 : avg forward pass time 12.70 +[Apr 27, 2026 - 13:46:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 168 : backward pass time 11.56 +[Apr 27, 2026 - 13:46:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 168 : optimizer step time 0.98 +[Apr 27, 2026 - 13:46:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 168-------------------- +[Apr 27, 2026 - 13:46:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0123 iter count: 1.0 +[Apr 27, 2026 - 13:46:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0123 iter count: 1.0 +[Apr 27, 2026 - 13:46:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:46:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 13:47:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076985 43.159908 42.986542 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.933594 70.175781 69.839844 +util 98.250000 100.000000 94.000000 +[Apr 27, 2026 - 13:47:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 169 : avg iteration time 50.37 seconds +[Apr 27, 2026 - 13:47:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 169 : data loading time 0.00 +[Apr 27, 2026 - 13:47:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 169 : avg forward pass time 12.56 +[Apr 27, 2026 - 13:47:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 169 : backward pass time 11.62 +[Apr 27, 2026 - 13:47:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 169 : optimizer step time 1.37 +[Apr 27, 2026 - 13:47:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 169-------------------- +[Apr 27, 2026 - 13:47:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0120 iter count: 1.0 +[Apr 27, 2026 - 13:47:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0120 iter count: 1.0 +[Apr 27, 2026 - 13:47:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:47:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 13:50:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076479 43.159912 42.986561 +peak_gpu_mem_gb 131.581049 131.620712 131.567828 +peak_gpu_mem_reserved_gb 135.547852 135.658203 135.458984 +util 70.000000 77.000000 66.000000 +[Apr 27, 2026 - 13:50:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 170 : avg iteration time 171.54 seconds +[Apr 27, 2026 - 13:50:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 170 : data loading time 0.00 +[Apr 27, 2026 - 13:50:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 170 : avg forward pass time 77.18 +[Apr 27, 2026 - 13:50:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 170 : backward pass time 8.42 +[Apr 27, 2026 - 13:50:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 170 : optimizer step time 0.34 +[Apr 27, 2026 - 13:50:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 170-------------------- +[Apr 27, 2026 - 13:50:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 1584544.3547 iter count: 1.0 +[Apr 27, 2026 - 13:50:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6917 iter count: 1.0 +[Apr 27, 2026 - 13:50:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 1584544.3547iter count: 1.0 +[Apr 27, 2026 - 13:50:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:50:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.1645iter count: 1.0 +[Apr 27, 2026 - 13:50:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.8292 iter count: 1.0 +[Apr 27, 2026 - 13:50:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.8493 iter count: 1.0 +[Apr 27, 2026 - 13:50:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 1885389.1250iter count: 1.0 +[Apr 27, 2026 - 13:50:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 5.2506 iter count: 1.0 +[Apr 27, 2026 - 13:50:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 48571588.0000iter count: 1.0 +[Apr 27, 2026 - 13:50:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6917iter count: 1.0 +[Apr 27, 2026 - 13:50:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 8.8493 iter count: 1.0 +[Apr 27, 2026 - 13:50:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 48571588.0000iter count: 1.0 +[Apr 27, 2026 - 13:50:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 6.0294 iter count: 1.0 +[Apr 27, 2026 - 13:50:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 172617.6250iter count: 1.0 +[Apr 27, 2026 - 13:50:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.3380 iter count: 1.0 +[Apr 27, 2026 - 13:50:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 5860.3042iter count: 1.0 +[Apr 27, 2026 - 13:50:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.5852 iter count: 1.0 +[Apr 27, 2026 - 13:50:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 1300.3977iter count: 1.0 +[Apr 27, 2026 - 13:50:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0134 iter count: 1.0 +[Apr 27, 2026 - 13:50:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:50:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 13:51:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076480 43.159916 42.986561 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.923828 70.000000 69.820312 +util 94.500000 99.000000 91.000000 +[Apr 27, 2026 - 13:51:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 171 : avg iteration time 51.55 seconds +[Apr 27, 2026 - 13:51:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 171 : data loading time 0.00 +[Apr 27, 2026 - 13:51:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 171 : avg forward pass time 13.25 +[Apr 27, 2026 - 13:51:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 171 : backward pass time 11.53 +[Apr 27, 2026 - 13:51:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 171 : optimizer step time 0.97 +[Apr 27, 2026 - 13:51:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 171-------------------- +[Apr 27, 2026 - 13:51:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0122 iter count: 1.0 +[Apr 27, 2026 - 13:51:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0122 iter count: 1.0 +[Apr 27, 2026 - 13:51:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:51:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 13:52:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076897 43.159893 42.986561 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.860352 69.980469 69.820312 +util 98.000000 99.000000 95.000000 +[Apr 27, 2026 - 13:52:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 172 : avg iteration time 51.12 seconds +[Apr 27, 2026 - 13:52:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 172 : data loading time 0.00 +[Apr 27, 2026 - 13:52:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 172 : avg forward pass time 12.83 +[Apr 27, 2026 - 13:52:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 172 : backward pass time 12.76 +[Apr 27, 2026 - 13:52:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 172 : optimizer step time 1.12 +[Apr 27, 2026 - 13:52:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 172-------------------- +[Apr 27, 2026 - 13:52:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0159 iter count: 1.0 +[Apr 27, 2026 - 13:52:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0159 iter count: 1.0 +[Apr 27, 2026 - 13:52:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:52:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 13:53:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076897 43.159893 42.986561 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.001953 70.039062 69.976562 +util 94.250000 99.000000 91.000000 +[Apr 27, 2026 - 13:53:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 173 : avg iteration time 49.68 seconds +[Apr 27, 2026 - 13:53:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 173 : data loading time 0.00 +[Apr 27, 2026 - 13:53:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 173 : avg forward pass time 12.76 +[Apr 27, 2026 - 13:53:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 173 : backward pass time 11.55 +[Apr 27, 2026 - 13:53:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 173 : optimizer step time 1.02 +[Apr 27, 2026 - 13:53:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 173-------------------- +[Apr 27, 2026 - 13:53:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0168 iter count: 1.0 +[Apr 27, 2026 - 13:53:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0168 iter count: 1.0 +[Apr 27, 2026 - 13:53:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:53:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 13:53:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076967 43.159908 42.986561 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.948242 70.039062 69.878906 +util 97.250000 100.000000 92.000000 +[Apr 27, 2026 - 13:53:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 174 : avg iteration time 51.55 seconds +[Apr 27, 2026 - 13:53:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 174 : data loading time 0.00 +[Apr 27, 2026 - 13:53:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 174 : avg forward pass time 13.29 +[Apr 27, 2026 - 13:53:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 174 : backward pass time 11.54 +[Apr 27, 2026 - 13:53:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 174 : optimizer step time 0.99 +[Apr 27, 2026 - 13:53:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 174-------------------- +[Apr 27, 2026 - 13:53:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0155 iter count: 1.0 +[Apr 27, 2026 - 13:53:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0155 iter count: 1.0 +[Apr 27, 2026 - 13:53:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:53:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 13:56:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076869 43.159901 42.986996 +peak_gpu_mem_gb 131.581049 131.620712 131.567828 +peak_gpu_mem_reserved_gb 135.566895 135.658203 135.457031 +util 71.500000 76.000000 64.000000 +[Apr 27, 2026 - 13:56:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 175 : avg iteration time 173.02 seconds +[Apr 27, 2026 - 13:56:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 175 : data loading time 0.00 +[Apr 27, 2026 - 13:56:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 175 : avg forward pass time 77.95 +[Apr 27, 2026 - 13:56:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 175 : backward pass time 8.43 +[Apr 27, 2026 - 13:56:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 175 : optimizer step time 0.34 +[Apr 27, 2026 - 13:56:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 175-------------------- +[Apr 27, 2026 - 13:56:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 607939.1768 iter count: 1.0 +[Apr 27, 2026 - 13:56:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6294 iter count: 1.0 +[Apr 27, 2026 - 13:56:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 607939.1768iter count: 1.0 +[Apr 27, 2026 - 13:56:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:56:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.6674iter count: 1.0 +[Apr 27, 2026 - 13:56:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.4472 iter count: 1.0 +[Apr 27, 2026 - 13:56:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.4317 iter count: 1.0 +[Apr 27, 2026 - 13:56:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 1253098.6250iter count: 1.0 +[Apr 27, 2026 - 13:56:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 2.4461 iter count: 1.0 +[Apr 27, 2026 - 13:56:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 21070816.0000iter count: 1.0 +[Apr 27, 2026 - 13:56:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6294iter count: 1.0 +[Apr 27, 2026 - 13:56:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.8165 iter count: 1.0 +[Apr 27, 2026 - 13:56:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 15261.4189iter count: 1.0 +[Apr 27, 2026 - 13:56:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.2353 iter count: 1.0 +[Apr 27, 2026 - 13:56:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 645.8417iter count: 1.0 +[Apr 27, 2026 - 13:56:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.6093 iter count: 1.0 +[Apr 27, 2026 - 13:56:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 74506.8984iter count: 1.0 +[Apr 27, 2026 - 13:56:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 2.5545 iter count: 1.0 +[Apr 27, 2026 - 13:56:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 165.5141iter count: 1.0 +[Apr 27, 2026 - 13:56:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0145 iter count: 1.0 +[Apr 27, 2026 - 13:56:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:56:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 13:57:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076868 43.159897 42.986996 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.865234 69.976562 69.546875 +util 96.750000 100.000000 91.000000 +[Apr 27, 2026 - 13:57:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 176 : avg iteration time 51.97 seconds +[Apr 27, 2026 - 13:57:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 176 : data loading time 0.00 +[Apr 27, 2026 - 13:57:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 176 : avg forward pass time 13.24 +[Apr 27, 2026 - 13:57:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 176 : backward pass time 11.54 +[Apr 27, 2026 - 13:57:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 176 : optimizer step time 0.98 +[Apr 27, 2026 - 13:57:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 176-------------------- +[Apr 27, 2026 - 13:57:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0154 iter count: 1.0 +[Apr 27, 2026 - 13:57:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0154 iter count: 1.0 +[Apr 27, 2026 - 13:57:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:57:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 13:58:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076978 43.159901 42.986996 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.899902 70.035156 69.566406 +util 97.250000 100.000000 94.000000 +[Apr 27, 2026 - 13:58:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 177 : avg iteration time 51.50 seconds +[Apr 27, 2026 - 13:58:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 177 : data loading time 0.00 +[Apr 27, 2026 - 13:58:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 177 : avg forward pass time 12.85 +[Apr 27, 2026 - 13:58:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 177 : backward pass time 12.75 +[Apr 27, 2026 - 13:58:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 177 : optimizer step time 1.50 +[Apr 27, 2026 - 13:58:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 177-------------------- +[Apr 27, 2026 - 13:58:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0164 iter count: 1.0 +[Apr 27, 2026 - 13:58:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0164 iter count: 1.0 +[Apr 27, 2026 - 13:58:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:58:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 13:59:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076982 43.159897 42.986996 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.987793 70.060547 69.898438 +util 97.750000 99.000000 94.000000 +[Apr 27, 2026 - 13:59:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 178 : avg iteration time 47.95 seconds +[Apr 27, 2026 - 13:59:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 178 : data loading time 0.00 +[Apr 27, 2026 - 13:59:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 178 : avg forward pass time 11.29 +[Apr 27, 2026 - 13:59:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 178 : backward pass time 12.75 +[Apr 27, 2026 - 13:59:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 178 : optimizer step time 1.05 +[Apr 27, 2026 - 13:59:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 178-------------------- +[Apr 27, 2026 - 13:59:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0145 iter count: 1.0 +[Apr 27, 2026 - 13:59:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0145 iter count: 1.0 +[Apr 27, 2026 - 13:59:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 13:59:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 14:00:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076977 43.159897 42.986996 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.026855 70.054688 69.962891 +util 97.500000 99.000000 96.000000 +[Apr 27, 2026 - 14:00:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 179 : avg iteration time 51.67 seconds +[Apr 27, 2026 - 14:00:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 179 : data loading time 0.00 +[Apr 27, 2026 - 14:00:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 179 : avg forward pass time 13.79 +[Apr 27, 2026 - 14:00:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 179 : backward pass time 11.52 +[Apr 27, 2026 - 14:00:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 179 : optimizer step time 0.99 +[Apr 27, 2026 - 14:00:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 179-------------------- +[Apr 27, 2026 - 14:00:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0147 iter count: 1.0 +[Apr 27, 2026 - 14:00:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0147 iter count: 1.0 +[Apr 27, 2026 - 14:00:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 14:00:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 14:03:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076887 43.160110 42.987122 +peak_gpu_mem_gb 131.581049 131.620712 131.567828 +peak_gpu_mem_reserved_gb 135.542969 135.658203 135.478516 +util 70.250000 77.000000 66.000000 +[Apr 27, 2026 - 14:03:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 180 : avg iteration time 176.60 seconds +[Apr 27, 2026 - 14:03:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 180 : data loading time 0.00 +[Apr 27, 2026 - 14:03:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 180 : avg forward pass time 79.74 +[Apr 27, 2026 - 14:03:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 180 : backward pass time 8.40 +[Apr 27, 2026 - 14:03:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 180 : optimizer step time 0.34 +[Apr 27, 2026 - 14:03:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 180-------------------- +[Apr 27, 2026 - 14:03:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 68735.7793 iter count: 1.0 +[Apr 27, 2026 - 14:03:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7075 iter count: 1.0 +[Apr 27, 2026 - 14:03:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 68735.7793iter count: 1.0 +[Apr 27, 2026 - 14:03:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 14:03:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.2377iter count: 1.0 +[Apr 27, 2026 - 14:03:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.4768 iter count: 1.0 +[Apr 27, 2026 - 14:03:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.1028 iter count: 1.0 +[Apr 27, 2026 - 14:03:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 131327.3281iter count: 1.0 +[Apr 27, 2026 - 14:03:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 2.5951 iter count: 1.0 +[Apr 27, 2026 - 14:03:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 1477025.3750iter count: 1.0 +[Apr 27, 2026 - 14:03:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7075iter count: 1.0 +[Apr 27, 2026 - 14:03:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.6679 iter count: 1.0 +[Apr 27, 2026 - 14:03:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 1534.4042iter count: 1.0 +[Apr 27, 2026 - 14:03:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.0517 iter count: 1.0 +[Apr 27, 2026 - 14:03:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 3305.8625iter count: 1.0 +[Apr 27, 2026 - 14:03:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 1.8746 iter count: 1.0 +[Apr 27, 2026 - 14:03:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 42.4879 iter count: 1.0 +[Apr 27, 2026 - 14:03:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.2810 iter count: 1.0 +[Apr 27, 2026 - 14:03:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 5228.7017iter count: 1.0 +[Apr 27, 2026 - 14:03:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0139 iter count: 1.0 +[Apr 27, 2026 - 14:03:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 14:03:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 14:03:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076957 43.160118 42.987125 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.080078 70.234375 69.937500 +util 97.500000 99.000000 96.000000 +[Apr 27, 2026 - 14:03:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 181 : avg iteration time 50.80 seconds +[Apr 27, 2026 - 14:03:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 181 : data loading time 0.00 +[Apr 27, 2026 - 14:03:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 181 : avg forward pass time 13.34 +[Apr 27, 2026 - 14:03:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 181 : backward pass time 11.54 +[Apr 27, 2026 - 14:03:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 181 : optimizer step time 0.98 +[Apr 27, 2026 - 14:03:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 181-------------------- +[Apr 27, 2026 - 14:03:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0164 iter count: 1.0 +[Apr 27, 2026 - 14:03:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0164 iter count: 1.0 +[Apr 27, 2026 - 14:03:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 14:03:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 14:04:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077009 43.160110 42.987129 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 71.286133 72.867188 69.332031 +util 97.500000 100.000000 92.000000 +[Apr 27, 2026 - 14:04:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 182 : avg iteration time 50.43 seconds +[Apr 27, 2026 - 14:04:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 182 : data loading time 0.00 +[Apr 27, 2026 - 14:04:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 182 : avg forward pass time 12.52 +[Apr 27, 2026 - 14:04:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 182 : backward pass time 11.58 +[Apr 27, 2026 - 14:04:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 182 : optimizer step time 1.10 +[Apr 27, 2026 - 14:04:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 182-------------------- +[Apr 27, 2026 - 14:04:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0138 iter count: 1.0 +[Apr 27, 2026 - 14:04:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0138 iter count: 1.0 +[Apr 27, 2026 - 14:04:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 14:04:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 14:05:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077010 43.160130 42.987122 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.997070 70.117188 69.937500 +util 97.250000 99.000000 93.000000 +[Apr 27, 2026 - 14:05:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 183 : avg iteration time 52.67 seconds +[Apr 27, 2026 - 14:05:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 183 : data loading time 0.00 +[Apr 27, 2026 - 14:05:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 183 : avg forward pass time 14.18 +[Apr 27, 2026 - 14:05:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 183 : backward pass time 11.75 +[Apr 27, 2026 - 14:05:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 183 : optimizer step time 0.98 +[Apr 27, 2026 - 14:05:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 183-------------------- +[Apr 27, 2026 - 14:05:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0137 iter count: 1.0 +[Apr 27, 2026 - 14:05:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0137 iter count: 1.0 +[Apr 27, 2026 - 14:05:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 14:05:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.02 seconds +[Apr 27, 2026 - 14:06:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077191 43.160110 42.987122 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.865234 70.078125 69.507812 +util 98.000000 99.000000 97.000000 +[Apr 27, 2026 - 14:06:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 184 : avg iteration time 46.91 seconds +[Apr 27, 2026 - 14:06:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 184 : data loading time 0.00 +[Apr 27, 2026 - 14:06:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 184 : avg forward pass time 11.32 +[Apr 27, 2026 - 14:06:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 184 : backward pass time 11.55 +[Apr 27, 2026 - 14:06:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 184 : optimizer step time 0.99 +[Apr 27, 2026 - 14:06:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 184-------------------- +[Apr 27, 2026 - 14:06:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0143 iter count: 1.0 +[Apr 27, 2026 - 14:06:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0143 iter count: 1.0 +[Apr 27, 2026 - 14:06:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 14:06:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 14:09:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076876 43.160110 42.987122 +peak_gpu_mem_gb 131.581049 131.620712 131.567828 +peak_gpu_mem_reserved_gb 135.513672 135.560547 135.419922 +util 68.250000 73.000000 60.000000 +[Apr 27, 2026 - 14:09:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 185 : avg iteration time 172.48 seconds +[Apr 27, 2026 - 14:09:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 185 : data loading time 0.00 +[Apr 27, 2026 - 14:09:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 185 : avg forward pass time 77.71 +[Apr 27, 2026 - 14:09:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 185 : backward pass time 8.37 +[Apr 27, 2026 - 14:09:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 185 : optimizer step time 0.35 +[Apr 27, 2026 - 14:09:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 185-------------------- +[Apr 27, 2026 - 14:09:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 210912.0884 iter count: 1.0 +[Apr 27, 2026 - 14:09:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6670 iter count: 1.0 +[Apr 27, 2026 - 14:09:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 210912.0884iter count: 1.0 +[Apr 27, 2026 - 14:09:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 14:09:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.3387iter count: 1.0 +[Apr 27, 2026 - 14:09:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.2916 iter count: 1.0 +[Apr 27, 2026 - 14:09:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.7965 iter count: 1.0 +[Apr 27, 2026 - 14:09:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 324522.7500iter count: 1.0 +[Apr 27, 2026 - 14:09:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 13.2400iter count: 1.0 +[Apr 27, 2026 - 14:09:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 5914640.5000iter count: 1.0 +[Apr 27, 2026 - 14:09:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6670iter count: 1.0 +[Apr 27, 2026 - 14:09:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.0670 iter count: 1.0 +[Apr 27, 2026 - 14:09:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 25185.4512iter count: 1.0 +[Apr 27, 2026 - 14:09:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.1389 iter count: 1.0 +[Apr 27, 2026 - 14:09:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 3935.5291iter count: 1.0 +[Apr 27, 2026 - 14:09:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.2313 iter count: 1.0 +[Apr 27, 2026 - 14:09:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 4734.7070iter count: 1.0 +[Apr 27, 2026 - 14:09:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 2.8416 iter count: 1.0 +[Apr 27, 2026 - 14:09:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 293.8740iter count: 1.0 +[Apr 27, 2026 - 14:09:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0159 iter count: 1.0 +[Apr 27, 2026 - 14:09:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 14:09:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 14:10:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076876 43.160130 42.987122 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.850586 70.214844 69.390625 +util 98.250000 100.000000 96.000000 +[Apr 27, 2026 - 14:10:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 186 : avg iteration time 55.72 seconds +[Apr 27, 2026 - 14:10:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 186 : data loading time 0.00 +[Apr 27, 2026 - 14:10:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 186 : avg forward pass time 15.01 +[Apr 27, 2026 - 14:10:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 186 : backward pass time 12.70 +[Apr 27, 2026 - 14:10:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 186 : optimizer step time 1.48 +[Apr 27, 2026 - 14:10:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 186-------------------- +[Apr 27, 2026 - 14:10:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0154 iter count: 1.0 +[Apr 27, 2026 - 14:10:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0154 iter count: 1.0 +[Apr 27, 2026 - 14:10:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 14:10:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 14:11:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076877 43.160110 42.987125 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.534180 72.417969 69.800781 +util 94.250000 99.000000 91.000000 +[Apr 27, 2026 - 14:11:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 187 : avg iteration time 47.94 seconds +[Apr 27, 2026 - 14:11:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 187 : data loading time 0.00 +[Apr 27, 2026 - 14:11:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 187 : avg forward pass time 11.27 +[Apr 27, 2026 - 14:11:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 187 : backward pass time 12.76 +[Apr 27, 2026 - 14:11:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 187 : optimizer step time 1.03 +[Apr 27, 2026 - 14:11:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 187-------------------- +[Apr 27, 2026 - 14:11:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0131 iter count: 1.0 +[Apr 27, 2026 - 14:11:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0131 iter count: 1.0 +[Apr 27, 2026 - 14:11:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 14:11:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 14:11:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076900 43.160110 42.987125 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.583008 72.222656 69.937500 +util 97.750000 99.000000 94.000000 +[Apr 27, 2026 - 14:11:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 188 : avg iteration time 47.11 seconds +[Apr 27, 2026 - 14:11:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 188 : data loading time 0.00 +[Apr 27, 2026 - 14:11:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 188 : avg forward pass time 11.52 +[Apr 27, 2026 - 14:11:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 188 : backward pass time 11.53 +[Apr 27, 2026 - 14:11:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 188 : optimizer step time 1.00 +[Apr 27, 2026 - 14:11:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 188-------------------- +[Apr 27, 2026 - 14:11:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0149 iter count: 1.0 +[Apr 27, 2026 - 14:11:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0149 iter count: 1.0 +[Apr 27, 2026 - 14:11:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 14:11:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 14:12:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076912 43.160110 42.987125 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.977539 70.097656 69.859375 +util 97.750000 99.000000 94.000000 +[Apr 27, 2026 - 14:12:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 189 : avg iteration time 53.46 seconds +[Apr 27, 2026 - 14:12:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 189 : data loading time 0.00 +[Apr 27, 2026 - 14:12:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 189 : avg forward pass time 14.05 +[Apr 27, 2026 - 14:12:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 189 : backward pass time 11.61 +[Apr 27, 2026 - 14:12:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 189 : optimizer step time 0.98 +[Apr 27, 2026 - 14:12:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 189-------------------- +[Apr 27, 2026 - 14:12:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0148 iter count: 1.0 +[Apr 27, 2026 - 14:12:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0148 iter count: 1.0 +[Apr 27, 2026 - 14:12:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 14:12:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 14:15:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076902 43.160110 42.987125 +peak_gpu_mem_gb 131.581049 131.620712 131.567828 +peak_gpu_mem_reserved_gb 135.542969 135.716797 135.419922 +util 70.500000 75.000000 63.000000 +[Apr 27, 2026 - 14:15:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 190 : avg iteration time 176.58 seconds +[Apr 27, 2026 - 14:15:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 190 : data loading time 0.00 +[Apr 27, 2026 - 14:15:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 190 : avg forward pass time 79.37 +[Apr 27, 2026 - 14:15:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 190 : backward pass time 8.47 +[Apr 27, 2026 - 14:15:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 190 : optimizer step time 0.34 +[Apr 27, 2026 - 14:15:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 190-------------------- +[Apr 27, 2026 - 14:15:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 544275.1924 iter count: 1.0 +[Apr 27, 2026 - 14:15:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6639 iter count: 1.0 +[Apr 27, 2026 - 14:15:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 544275.1924iter count: 1.0 +[Apr 27, 2026 - 14:15:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 14:15:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.8367iter count: 1.0 +[Apr 27, 2026 - 14:15:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.4930 iter count: 1.0 +[Apr 27, 2026 - 14:15:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.0430 iter count: 1.0 +[Apr 27, 2026 - 14:15:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 736134.5000iter count: 1.0 +[Apr 27, 2026 - 14:15:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 19.8054iter count: 1.0 +[Apr 27, 2026 - 14:15:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 9684606.0000iter count: 1.0 +[Apr 27, 2026 - 14:15:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6639iter count: 1.0 +[Apr 27, 2026 - 14:15:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.8160 iter count: 1.0 +[Apr 27, 2026 - 14:15:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 112634.2109iter count: 1.0 +[Apr 27, 2026 - 14:15:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.4062 iter count: 1.0 +[Apr 27, 2026 - 14:15:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 909.0455iter count: 1.0 +[Apr 27, 2026 - 14:15:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 2.8662 iter count: 1.0 +[Apr 27, 2026 - 14:15:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 308.6848iter count: 1.0 +[Apr 27, 2026 - 14:15:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.3227 iter count: 1.0 +[Apr 27, 2026 - 14:15:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 5683.8921iter count: 1.0 +[Apr 27, 2026 - 14:15:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0149 iter count: 1.0 +[Apr 27, 2026 - 14:15:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 14:15:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 14:16:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076911 43.160126 42.987125 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.558594 72.417969 69.902344 +util 96.500000 99.000000 91.000000 +[Apr 27, 2026 - 14:16:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 191 : avg iteration time 53.92 seconds +[Apr 27, 2026 - 14:16:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 191 : data loading time 0.00 +[Apr 27, 2026 - 14:16:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 191 : avg forward pass time 14.77 +[Apr 27, 2026 - 14:16:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 191 : backward pass time 11.57 +[Apr 27, 2026 - 14:16:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 191 : optimizer step time 0.98 +[Apr 27, 2026 - 14:16:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 191-------------------- +[Apr 27, 2026 - 14:16:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0177 iter count: 1.0 +[Apr 27, 2026 - 14:16:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0177 iter count: 1.0 +[Apr 27, 2026 - 14:16:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 14:16:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 14:17:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076908 43.160110 42.987125 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.865234 69.996094 69.546875 +util 98.000000 99.000000 97.000000 +[Apr 27, 2026 - 14:17:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 192 : avg iteration time 47.39 seconds +[Apr 27, 2026 - 14:17:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 192 : data loading time 0.00 +[Apr 27, 2026 - 14:17:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 192 : avg forward pass time 11.18 +[Apr 27, 2026 - 14:17:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 192 : backward pass time 11.53 +[Apr 27, 2026 - 14:17:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 192 : optimizer step time 1.02 +[Apr 27, 2026 - 14:17:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 192-------------------- +[Apr 27, 2026 - 14:17:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0146 iter count: 1.0 +[Apr 27, 2026 - 14:17:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0146 iter count: 1.0 +[Apr 27, 2026 - 14:17:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 14:17:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 14:18:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076907 43.160122 42.987129 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.894531 70.019531 69.566406 +util 95.750000 100.000000 92.000000 +[Apr 27, 2026 - 14:18:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 193 : avg iteration time 53.15 seconds +[Apr 27, 2026 - 14:18:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 193 : data loading time 0.00 +[Apr 27, 2026 - 14:18:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 193 : avg forward pass time 14.12 +[Apr 27, 2026 - 14:18:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 193 : backward pass time 12.35 +[Apr 27, 2026 - 14:18:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 193 : optimizer step time 1.01 +[Apr 27, 2026 - 14:18:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 193-------------------- +[Apr 27, 2026 - 14:18:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0136 iter count: 1.0 +[Apr 27, 2026 - 14:18:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0136 iter count: 1.0 +[Apr 27, 2026 - 14:18:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 14:18:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 14:19:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076913 43.160122 42.987129 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.948242 69.996094 69.878906 +util 97.750000 99.000000 94.000000 +[Apr 27, 2026 - 14:19:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 194 : avg iteration time 52.09 seconds +[Apr 27, 2026 - 14:19:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 194 : data loading time 0.00 +[Apr 27, 2026 - 14:19:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 194 : avg forward pass time 13.98 +[Apr 27, 2026 - 14:19:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 194 : backward pass time 11.56 +[Apr 27, 2026 - 14:19:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 194 : optimizer step time 0.98 +[Apr 27, 2026 - 14:19:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 194-------------------- +[Apr 27, 2026 - 14:19:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0136 iter count: 1.0 +[Apr 27, 2026 - 14:19:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0136 iter count: 1.0 +[Apr 27, 2026 - 14:19:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 14:19:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 14:21:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076568 43.160118 42.987129 +peak_gpu_mem_gb 131.581049 131.620712 131.567828 +peak_gpu_mem_reserved_gb 135.571289 135.638672 135.458984 +util 68.750000 76.000000 65.000000 +[Apr 27, 2026 - 14:21:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 195 : avg iteration time 168.93 seconds +[Apr 27, 2026 - 14:21:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 195 : data loading time 0.00 +[Apr 27, 2026 - 14:21:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 195 : avg forward pass time 75.95 +[Apr 27, 2026 - 14:21:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 195 : backward pass time 8.38 +[Apr 27, 2026 - 14:21:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 195 : optimizer step time 0.34 +[Apr 27, 2026 - 14:21:54 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 195-------------------- +[Apr 27, 2026 - 14:21:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 404558.5781 iter count: 1.0 +[Apr 27, 2026 - 14:21:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6300 iter count: 1.0 +[Apr 27, 2026 - 14:21:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 404558.5781iter count: 1.0 +[Apr 27, 2026 - 14:21:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 14:21:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.8630iter count: 1.0 +[Apr 27, 2026 - 14:21:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.5259 iter count: 1.0 +[Apr 27, 2026 - 14:21:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.1067 iter count: 1.0 +[Apr 27, 2026 - 14:21:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 755342.8750iter count: 1.0 +[Apr 27, 2026 - 14:21:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 2.8629 iter count: 1.0 +[Apr 27, 2026 - 14:21:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 11000963.0000iter count: 1.0 +[Apr 27, 2026 - 14:21:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6300iter count: 1.0 +[Apr 27, 2026 - 14:21:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.6681 iter count: 1.0 +[Apr 27, 2026 - 14:21:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 1534.9515iter count: 1.0 +[Apr 27, 2026 - 14:21:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.0622 iter count: 1.0 +[Apr 27, 2026 - 14:21:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 456.8257iter count: 1.0 +[Apr 27, 2026 - 14:21:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 6.8651 iter count: 1.0 +[Apr 27, 2026 - 14:21:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 918260.1875iter count: 1.0 +[Apr 27, 2026 - 14:21:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 8.1067 iter count: 1.0 +[Apr 27, 2026 - 14:21:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 11000963.0000iter count: 1.0 +[Apr 27, 2026 - 14:21:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0134 iter count: 1.0 +[Apr 27, 2026 - 14:21:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 14:21:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 14:22:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076563 43.160130 42.987129 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.982422 70.156250 69.878906 +util 98.500000 100.000000 97.000000 +[Apr 27, 2026 - 14:22:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 196 : avg iteration time 55.45 seconds +[Apr 27, 2026 - 14:22:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 196 : data loading time 0.00 +[Apr 27, 2026 - 14:22:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 196 : avg forward pass time 14.92 +[Apr 27, 2026 - 14:22:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 196 : backward pass time 12.88 +[Apr 27, 2026 - 14:22:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 196 : optimizer step time 1.21 +[Apr 27, 2026 - 14:22:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 196-------------------- +[Apr 27, 2026 - 14:22:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0132 iter count: 1.0 +[Apr 27, 2026 - 14:22:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0132 iter count: 1.0 +[Apr 27, 2026 - 14:22:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 14:22:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 14:23:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076874 43.160114 42.987129 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.938477 70.214844 69.781250 +util 97.750000 99.000000 94.000000 +[Apr 27, 2026 - 14:23:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 197 : avg iteration time 52.42 seconds +[Apr 27, 2026 - 14:23:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 197 : data loading time 0.00 +[Apr 27, 2026 - 14:23:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 197 : avg forward pass time 14.18 +[Apr 27, 2026 - 14:23:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 197 : backward pass time 11.54 +[Apr 27, 2026 - 14:23:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 197 : optimizer step time 0.98 +[Apr 27, 2026 - 14:23:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 197-------------------- +[Apr 27, 2026 - 14:23:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0140 iter count: 1.0 +[Apr 27, 2026 - 14:23:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0140 iter count: 1.0 +[Apr 27, 2026 - 14:23:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 14:23:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 14:24:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076889 43.160152 42.987129 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.928711 70.078125 69.800781 +util 96.750000 100.000000 94.000000 +[Apr 27, 2026 - 14:24:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 198 : avg iteration time 50.53 seconds +[Apr 27, 2026 - 14:24:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 198 : data loading time 0.00 +[Apr 27, 2026 - 14:24:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 198 : avg forward pass time 12.52 +[Apr 27, 2026 - 14:24:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 198 : backward pass time 11.59 +[Apr 27, 2026 - 14:24:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 198 : optimizer step time 0.98 +[Apr 27, 2026 - 14:24:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 198-------------------- +[Apr 27, 2026 - 14:24:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0121 iter count: 1.0 +[Apr 27, 2026 - 14:24:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[Apr 27, 2026 - 14:24:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 14:24:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 14:25:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076813 43.160114 42.987129 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.923828 70.117188 69.781250 +util 97.250000 100.000000 92.000000 +[Apr 27, 2026 - 14:25:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 199 : avg iteration time 50.91 seconds +[Apr 27, 2026 - 14:25:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 199 : data loading time 0.00 +[Apr 27, 2026 - 14:25:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 199 : avg forward pass time 12.74 +[Apr 27, 2026 - 14:25:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 199 : backward pass time 11.91 +[Apr 27, 2026 - 14:25:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 199 : optimizer step time 1.54 +[Apr 27, 2026 - 14:25:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 199-------------------- +[Apr 27, 2026 - 14:25:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0143 iter count: 1.0 +[Apr 27, 2026 - 14:25:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0143 iter count: 1.0 +[Apr 27, 2026 - 14:25:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 14:25:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 14:28:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.076601 43.160114 42.987129 +peak_gpu_mem_gb 131.581049 131.620712 131.567828 +peak_gpu_mem_reserved_gb 135.572266 135.619141 135.439453 +util 67.500000 75.000000 63.000000 +[Apr 27, 2026 - 14:28:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 200 : avg iteration time 175.90 seconds +[Apr 27, 2026 - 14:28:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 200 : data loading time 0.00 +[Apr 27, 2026 - 14:28:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 200 : avg forward pass time 78.93 +[Apr 27, 2026 - 14:28:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 200 : backward pass time 8.41 +[Apr 27, 2026 - 14:28:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 200 : optimizer step time 0.36 +[Apr 27, 2026 - 14:28:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 200-------------------- +[Apr 27, 2026 - 14:28:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 87506.8711 iter count: 1.0 +[Apr 27, 2026 - 14:28:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6491 iter count: 1.0 +[Apr 27, 2026 - 14:28:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 87506.8711iter count: 1.0 +[Apr 27, 2026 - 14:28:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 14:28:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.6897iter count: 1.0 +[Apr 27, 2026 - 14:28:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.4073 iter count: 1.0 +[Apr 27, 2026 - 14:28:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.2363 iter count: 1.0 +[Apr 27, 2026 - 14:28:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 159005.4531iter count: 1.0 +[Apr 27, 2026 - 14:28:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 2.2583 iter count: 1.0 +[Apr 27, 2026 - 14:28:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 1929045.8750iter count: 1.0 +[Apr 27, 2026 - 14:28:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6491iter count: 1.0 +[Apr 27, 2026 - 14:28:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.8765 iter count: 1.0 +[Apr 27, 2026 - 14:28:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 2328.3306iter count: 1.0 +[Apr 27, 2026 - 14:28:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.2917 iter count: 1.0 +[Apr 27, 2026 - 14:28:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 5342.0791iter count: 1.0 +[Apr 27, 2026 - 14:28:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.1739 iter count: 1.0 +[Apr 27, 2026 - 14:28:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 4221.0991iter count: 1.0 +[Apr 27, 2026 - 14:28:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.4173 iter count: 1.0 +[Apr 27, 2026 - 14:28:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 929.3647iter count: 1.0 +[Apr 27, 2026 - 14:28:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0145 iter count: 1.0 +[Apr 27, 2026 - 14:28:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[Apr 27, 2026 - 14:28:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 14.99 seconds +[Apr 27, 2026 - 14:28:35 | INFO | fastgen.trainer:save_checkpoint:266 ] Saving checkpoint iteration 200 +[Apr 27, 2026 - 14:28:35 | INFO | fastgen.utils.checkpointer:save:301 ] Saving FSDP model to prefix /tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000200 +[Apr 27, 2026 - 14:28:48 | SUCCESS | fastgen.utils.checkpointer:save:329 ] Model saved at iteration 200 +[Apr 27, 2026 - 14:28:48 | INFO | fastgen.trainer:validate:358 ] Validation iteration 200 +[Apr 27, 2026 - 14:28:49 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 14:28:51 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 2.2s +[Apr 27, 2026 - 14:28:54 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 0: sync_c=6.234 +[Apr 27, 2026 - 14:28:55 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 14:28:56 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.8s +[Apr 27, 2026 - 14:28:59 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 1: sync_c=5.713 +[Apr 27, 2026 - 14:29:00 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 14:29:01 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.8s +[Apr 27, 2026 - 14:29:04 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 2: sync_c=4.662 +[Apr 27, 2026 - 14:29:05 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 14:29:06 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.8s +[Apr 27, 2026 - 14:29:09 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 3: sync_c=4.907 +[Apr 27, 2026 - 14:29:10 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 14:29:12 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.8s +[Apr 27, 2026 - 14:29:15 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 4: sync_c=4.648 +[Apr 27, 2026 - 14:29:15 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 14:29:17 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.8s +[Apr 27, 2026 - 14:29:20 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 5: sync_c=6.939 +[Apr 27, 2026 - 14:29:20 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 14:29:22 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.8s +[Apr 27, 2026 - 14:29:25 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 6: sync_c=6.658 +[Apr 27, 2026 - 14:29:25 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 14:29:27 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.8s +[Apr 27, 2026 - 14:29:30 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 7: sync_c=4.891 +[Apr 27, 2026 - 14:29:30 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 14:29:32 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.8s +[Apr 27, 2026 - 14:29:35 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 8: sync_c=5.817 +[Apr 27, 2026 - 14:29:35 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 14:29:37 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.8s +[Apr 27, 2026 - 14:29:40 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 9: sync_c=2.869 +[Apr 27, 2026 - 14:29:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging val0 stats at iteration 200-------------------- +[Apr 27, 2026 - 14:29:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0000 iter count: 10.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[Apr 27, 2026 - 14:29:57 | INFO | fastgen.callbacks.wandb:on_validation_end:763 ] Logged 10 val videos at iteration 200 +[Apr 27, 2026 - 14:29:57 | INFO | fastgen.callbacks.wandb:on_validation_end:769 ] [SyncEval] val0 mean sync_c: 5.334 (n=10) +[Apr 27, 2026 - 14:30:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077707 43.159878 42.986614 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.129395 70.509766 69.976562 +util 97.250000 100.000000 92.000000 +[Apr 27, 2026 - 14:30:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 201 : avg iteration time 146.71 seconds +[Apr 27, 2026 - 14:30:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 201 : data loading time 0.00 +[Apr 27, 2026 - 14:30:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 201 : avg forward pass time 12.17 +[Apr 27, 2026 - 14:30:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 201 : backward pass time 12.67 +[Apr 27, 2026 - 14:30:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 201 : optimizer step time 1.02 +[Apr 27, 2026 - 14:30:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 201-------------------- +[Apr 27, 2026 - 14:30:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0119 iter count: 1.0 +[Apr 27, 2026 - 14:30:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[Apr 27, 2026 - 14:30:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 14:30:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 14:31:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077703 43.159878 42.986614 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.148926 70.509766 69.957031 +util 97.250000 100.000000 92.000000 +[Apr 27, 2026 - 14:31:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 202 : avg iteration time 51.79 seconds +[Apr 27, 2026 - 14:31:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 202 : data loading time 0.00 +[Apr 27, 2026 - 14:31:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 202 : avg forward pass time 13.81 +[Apr 27, 2026 - 14:31:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 202 : backward pass time 11.62 +[Apr 27, 2026 - 14:31:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 202 : optimizer step time 0.97 +[Apr 27, 2026 - 14:31:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 202-------------------- +[Apr 27, 2026 - 14:31:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0138 iter count: 1.0 +[Apr 27, 2026 - 14:31:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0138 iter count: 1.0 +[Apr 27, 2026 - 14:31:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 14:31:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 14:32:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077248 43.159901 42.986614 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.173340 70.509766 70.015625 +util 98.250000 100.000000 95.000000 +[Apr 27, 2026 - 14:32:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 203 : avg iteration time 50.38 seconds +[Apr 27, 2026 - 14:32:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 203 : data loading time 0.00 +[Apr 27, 2026 - 14:32:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 203 : avg forward pass time 12.56 +[Apr 27, 2026 - 14:32:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 203 : backward pass time 11.58 +[Apr 27, 2026 - 14:32:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 203 : optimizer step time 1.23 +[Apr 27, 2026 - 14:32:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 203-------------------- +[Apr 27, 2026 - 14:32:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0123 iter count: 1.0 +[Apr 27, 2026 - 14:32:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0123 iter count: 1.0 +[Apr 27, 2026 - 14:32:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 14:32:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 14:33:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077299 43.159878 42.986637 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.153809 70.509766 69.957031 +util 96.250000 100.000000 90.000000 +[Apr 27, 2026 - 14:33:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 204 : avg iteration time 47.81 seconds +[Apr 27, 2026 - 14:33:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 204 : data loading time 0.00 +[Apr 27, 2026 - 14:33:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 204 : avg forward pass time 11.12 +[Apr 27, 2026 - 14:33:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 204 : backward pass time 12.90 +[Apr 27, 2026 - 14:33:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 204 : optimizer step time 1.11 +[Apr 27, 2026 - 14:33:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 204-------------------- +[Apr 27, 2026 - 14:33:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0108 iter count: 1.0 +[Apr 27, 2026 - 14:33:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[Apr 27, 2026 - 14:33:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 14:33:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 14:36:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077382 43.160351 42.987080 +peak_gpu_mem_gb 131.581049 131.620712 131.567828 +peak_gpu_mem_reserved_gb 135.512695 135.580078 135.457031 +util 69.750000 73.000000 66.000000 +[Apr 27, 2026 - 14:36:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 205 : avg iteration time 174.19 seconds +[Apr 27, 2026 - 14:36:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 205 : data loading time 0.00 +[Apr 27, 2026 - 14:36:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 205 : avg forward pass time 78.48 +[Apr 27, 2026 - 14:36:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 205 : backward pass time 8.54 +[Apr 27, 2026 - 14:36:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 205 : optimizer step time 0.34 +[Apr 27, 2026 - 14:36:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 205-------------------- +[Apr 27, 2026 - 14:36:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 194468.9082 iter count: 1.0 +[Apr 27, 2026 - 14:36:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6341 iter count: 1.0 +[Apr 27, 2026 - 14:36:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 194468.9082iter count: 1.0 +[Apr 27, 2026 - 14:36:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 14:36:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.2332iter count: 1.0 +[Apr 27, 2026 - 14:36:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.5057 iter count: 1.0 +[Apr 27, 2026 - 14:36:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.3930 iter count: 1.0 +[Apr 27, 2026 - 14:36:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 254334.5000iter count: 1.0 +[Apr 27, 2026 - 14:36:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 2.7496 iter count: 1.0 +[Apr 27, 2026 - 14:36:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 2639120.7500iter count: 1.0 +[Apr 27, 2026 - 14:36:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6341iter count: 1.0 +[Apr 27, 2026 - 14:36:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 6.5547 iter count: 1.0 +[Apr 27, 2026 - 14:36:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 493528.0312iter count: 1.0 +[Apr 27, 2026 - 14:36:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 2.2960 iter count: 1.0 +[Apr 27, 2026 - 14:36:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 98.6872 iter count: 1.0 +[Apr 27, 2026 - 14:36:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.9421 iter count: 1.0 +[Apr 27, 2026 - 14:36:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 19618.2637iter count: 1.0 +[Apr 27, 2026 - 14:36:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.1157 iter count: 1.0 +[Apr 27, 2026 - 14:36:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 27760.4355iter count: 1.0 +[Apr 27, 2026 - 14:36:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0127 iter count: 1.0 +[Apr 27, 2026 - 14:36:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 14:36:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 14:37:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077438 43.160358 42.987080 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.879883 70.000000 69.820312 +util 97.750000 99.000000 96.000000 +[Apr 27, 2026 - 14:37:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 206 : avg iteration time 52.14 seconds +[Apr 27, 2026 - 14:37:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 206 : data loading time 0.00 +[Apr 27, 2026 - 14:37:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 206 : avg forward pass time 13.47 +[Apr 27, 2026 - 14:37:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 206 : backward pass time 11.58 +[Apr 27, 2026 - 14:37:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 206 : optimizer step time 0.98 +[Apr 27, 2026 - 14:37:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 206-------------------- +[Apr 27, 2026 - 14:37:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0156 iter count: 1.0 +[Apr 27, 2026 - 14:37:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0156 iter count: 1.0 +[Apr 27, 2026 - 14:37:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 14:37:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 14:37:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077746 43.160351 42.987087 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.845703 69.917969 69.648438 +util 97.500000 100.000000 93.000000 +[Apr 27, 2026 - 14:37:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 207 : avg iteration time 50.69 seconds +[Apr 27, 2026 - 14:37:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 207 : data loading time 0.00 +[Apr 27, 2026 - 14:37:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 207 : avg forward pass time 12.85 +[Apr 27, 2026 - 14:37:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 207 : backward pass time 11.86 +[Apr 27, 2026 - 14:37:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 207 : optimizer step time 1.53 +[Apr 27, 2026 - 14:37:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 207-------------------- +[Apr 27, 2026 - 14:37:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0133 iter count: 1.0 +[Apr 27, 2026 - 14:37:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0133 iter count: 1.0 +[Apr 27, 2026 - 14:37:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 14:37:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 14:38:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077850 43.160351 42.987087 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.884766 69.976562 69.765625 +util 97.250000 99.000000 95.000000 +[Apr 27, 2026 - 14:38:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 208 : avg iteration time 50.31 seconds +[Apr 27, 2026 - 14:38:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 208 : data loading time 0.00 +[Apr 27, 2026 - 14:38:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 208 : avg forward pass time 12.67 +[Apr 27, 2026 - 14:38:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 208 : backward pass time 12.38 +[Apr 27, 2026 - 14:38:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 208 : optimizer step time 1.01 +[Apr 27, 2026 - 14:38:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 208-------------------- +[Apr 27, 2026 - 14:38:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0125 iter count: 1.0 +[Apr 27, 2026 - 14:38:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0125 iter count: 1.0 +[Apr 27, 2026 - 14:38:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 14:38:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 14:39:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077844 43.160351 42.987064 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.855469 69.898438 69.785156 +util 98.250000 99.000000 97.000000 +[Apr 27, 2026 - 14:39:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 209 : avg iteration time 52.01 seconds +[Apr 27, 2026 - 14:39:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 209 : data loading time 0.00 +[Apr 27, 2026 - 14:39:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 209 : avg forward pass time 13.93 +[Apr 27, 2026 - 14:39:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 209 : backward pass time 11.57 +[Apr 27, 2026 - 14:39:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 209 : optimizer step time 0.98 +[Apr 27, 2026 - 14:39:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 209-------------------- +[Apr 27, 2026 - 14:39:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0141 iter count: 1.0 +[Apr 27, 2026 - 14:39:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0141 iter count: 1.0 +[Apr 27, 2026 - 14:39:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 14:39:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 14:42:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077390 43.160351 42.987068 +peak_gpu_mem_gb 131.581049 131.620712 131.567828 +peak_gpu_mem_reserved_gb 135.572266 135.716797 135.517578 +util 78.750000 92.000000 68.000000 +[Apr 27, 2026 - 14:42:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 210 : avg iteration time 172.26 seconds +[Apr 27, 2026 - 14:42:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 210 : data loading time 0.00 +[Apr 27, 2026 - 14:42:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 210 : avg forward pass time 77.18 +[Apr 27, 2026 - 14:42:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 210 : backward pass time 9.25 +[Apr 27, 2026 - 14:42:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 210 : optimizer step time 0.34 +[Apr 27, 2026 - 14:42:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 210-------------------- +[Apr 27, 2026 - 14:42:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 3118658.0195 iter count: 1.0 +[Apr 27, 2026 - 14:42:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6465 iter count: 1.0 +[Apr 27, 2026 - 14:42:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 3118658.0195iter count: 1.0 +[Apr 27, 2026 - 14:42:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 14:42:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.7423iter count: 1.0 +[Apr 27, 2026 - 14:42:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.3013 iter count: 1.0 +[Apr 27, 2026 - 14:42:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 9.5307 iter count: 1.0 +[Apr 27, 2026 - 14:42:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 6010200.5000iter count: 1.0 +[Apr 27, 2026 - 14:42:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 1.8267 iter count: 1.0 +[Apr 27, 2026 - 14:42:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 189779728.0000iter count: 1.0 +[Apr 27, 2026 - 14:42:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6465iter count: 1.0 +[Apr 27, 2026 - 14:42:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.4813 iter count: 1.0 +[Apr 27, 2026 - 14:42:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 7805.1104iter count: 1.0 +[Apr 27, 2026 - 14:42:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.5364 iter count: 1.0 +[Apr 27, 2026 - 14:42:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 8715.1055iter count: 1.0 +[Apr 27, 2026 - 14:42:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.3332 iter count: 1.0 +[Apr 27, 2026 - 14:42:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 42889.4805iter count: 1.0 +[Apr 27, 2026 - 14:42:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 2.6404 iter count: 1.0 +[Apr 27, 2026 - 14:42:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 196.5263iter count: 1.0 +[Apr 27, 2026 - 14:42:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0122 iter count: 1.0 +[Apr 27, 2026 - 14:42:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 14:42:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 14:43:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077389 43.160351 42.987064 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.977539 70.019531 69.898438 +util 96.000000 99.000000 93.000000 +[Apr 27, 2026 - 14:43:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 211 : avg iteration time 51.70 seconds +[Apr 27, 2026 - 14:43:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 211 : data loading time 0.00 +[Apr 27, 2026 - 14:43:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 211 : avg forward pass time 13.41 +[Apr 27, 2026 - 14:43:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 211 : backward pass time 12.33 +[Apr 27, 2026 - 14:43:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 211 : optimizer step time 0.99 +[Apr 27, 2026 - 14:43:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 211-------------------- +[Apr 27, 2026 - 14:43:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0128 iter count: 1.0 +[Apr 27, 2026 - 14:43:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0128 iter count: 1.0 +[Apr 27, 2026 - 14:43:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 14:43:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 14:44:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077638 43.160366 42.987064 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.943359 69.996094 69.859375 +util 97.750000 99.000000 94.000000 +[Apr 27, 2026 - 14:44:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 212 : avg iteration time 52.08 seconds +[Apr 27, 2026 - 14:44:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 212 : data loading time 0.00 +[Apr 27, 2026 - 14:44:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 212 : avg forward pass time 13.96 +[Apr 27, 2026 - 14:44:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 212 : backward pass time 11.56 +[Apr 27, 2026 - 14:44:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 212 : optimizer step time 0.99 +[Apr 27, 2026 - 14:44:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 212-------------------- +[Apr 27, 2026 - 14:44:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0136 iter count: 1.0 +[Apr 27, 2026 - 14:44:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0136 iter count: 1.0 +[Apr 27, 2026 - 14:44:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 14:44:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 14:44:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077644 43.160370 42.987087 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.982422 70.019531 69.878906 +util 98.000000 100.000000 96.000000 +[Apr 27, 2026 - 14:44:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 213 : avg iteration time 47.55 seconds +[Apr 27, 2026 - 14:44:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 213 : data loading time 0.00 +[Apr 27, 2026 - 14:44:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 213 : avg forward pass time 11.19 +[Apr 27, 2026 - 14:44:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 213 : backward pass time 11.53 +[Apr 27, 2026 - 14:44:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 213 : optimizer step time 0.98 +[Apr 27, 2026 - 14:44:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 213-------------------- +[Apr 27, 2026 - 14:44:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0131 iter count: 1.0 +[Apr 27, 2026 - 14:44:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0131 iter count: 1.0 +[Apr 27, 2026 - 14:44:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 14:44:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 14:45:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077700 43.160362 42.987064 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.943359 69.996094 69.859375 +util 97.250000 100.000000 92.000000 +[Apr 27, 2026 - 14:45:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 214 : avg iteration time 54.43 seconds +[Apr 27, 2026 - 14:45:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 214 : data loading time 0.00 +[Apr 27, 2026 - 14:45:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 214 : avg forward pass time 14.31 +[Apr 27, 2026 - 14:45:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 214 : backward pass time 13.06 +[Apr 27, 2026 - 14:45:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 214 : optimizer step time 1.21 +[Apr 27, 2026 - 14:45:54 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 214-------------------- +[Apr 27, 2026 - 14:45:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0139 iter count: 1.0 +[Apr 27, 2026 - 14:45:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0139 iter count: 1.0 +[Apr 27, 2026 - 14:45:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 14:45:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 14:48:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077454 43.160374 42.987064 +peak_gpu_mem_gb 131.581049 131.620712 131.567828 +peak_gpu_mem_reserved_gb 135.567383 135.697266 135.458984 +util 71.000000 78.000000 67.000000 +[Apr 27, 2026 - 14:48:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 215 : avg iteration time 172.57 seconds +[Apr 27, 2026 - 14:48:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 215 : data loading time 0.00 +[Apr 27, 2026 - 14:48:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 215 : avg forward pass time 77.64 +[Apr 27, 2026 - 14:48:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 215 : backward pass time 8.45 +[Apr 27, 2026 - 14:48:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 215 : optimizer step time 0.34 +[Apr 27, 2026 - 14:48:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 215-------------------- +[Apr 27, 2026 - 14:48:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 150763.7451 iter count: 1.0 +[Apr 27, 2026 - 14:48:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6992 iter count: 1.0 +[Apr 27, 2026 - 14:48:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 150763.7451iter count: 1.0 +[Apr 27, 2026 - 14:48:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 14:48:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.7426iter count: 1.0 +[Apr 27, 2026 - 14:48:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.7326 iter count: 1.0 +[Apr 27, 2026 - 14:48:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.1911 iter count: 1.0 +[Apr 27, 2026 - 14:48:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 219668.7188iter count: 1.0 +[Apr 27, 2026 - 14:48:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 4.3287 iter count: 1.0 +[Apr 27, 2026 - 14:48:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 1762473.6250iter count: 1.0 +[Apr 27, 2026 - 14:48:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6992iter count: 1.0 +[Apr 27, 2026 - 14:48:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.9918 iter count: 1.0 +[Apr 27, 2026 - 14:48:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 160112.3594iter count: 1.0 +[Apr 27, 2026 - 14:48:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.5819 iter count: 1.0 +[Apr 27, 2026 - 14:48:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 9545.3174iter count: 1.0 +[Apr 27, 2026 - 14:48:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 1.5872 iter count: 1.0 +[Apr 27, 2026 - 14:48:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 23.9101 iter count: 1.0 +[Apr 27, 2026 - 14:48:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.7623 iter count: 1.0 +[Apr 27, 2026 - 14:48:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 101173.8125iter count: 1.0 +[Apr 27, 2026 - 14:48:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0139 iter count: 1.0 +[Apr 27, 2026 - 14:48:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 14:48:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 14:49:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077448 43.160351 42.987064 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.948730 70.041016 69.761719 +util 94.750000 99.000000 91.000000 +[Apr 27, 2026 - 14:49:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 216 : avg iteration time 52.34 seconds +[Apr 27, 2026 - 14:49:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 216 : data loading time 0.00 +[Apr 27, 2026 - 14:49:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 216 : avg forward pass time 13.39 +[Apr 27, 2026 - 14:49:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 216 : backward pass time 11.58 +[Apr 27, 2026 - 14:49:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 216 : optimizer step time 0.98 +[Apr 27, 2026 - 14:49:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 216-------------------- +[Apr 27, 2026 - 14:49:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0125 iter count: 1.0 +[Apr 27, 2026 - 14:49:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0125 iter count: 1.0 +[Apr 27, 2026 - 14:49:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 14:49:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 14:50:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077459 43.160351 42.987087 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.929199 70.015625 69.761719 +util 97.000000 100.000000 92.000000 +[Apr 27, 2026 - 14:50:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 217 : avg iteration time 51.14 seconds +[Apr 27, 2026 - 14:50:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 217 : data loading time 0.00 +[Apr 27, 2026 - 14:50:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 217 : avg forward pass time 12.86 +[Apr 27, 2026 - 14:50:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 217 : backward pass time 12.16 +[Apr 27, 2026 - 14:50:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 217 : optimizer step time 1.57 +[Apr 27, 2026 - 14:50:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 217-------------------- +[Apr 27, 2026 - 14:50:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0158 iter count: 1.0 +[Apr 27, 2026 - 14:50:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0158 iter count: 1.0 +[Apr 27, 2026 - 14:50:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 14:50:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 14:51:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077460 43.160374 42.987064 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.924316 69.996094 69.781250 +util 96.250000 99.000000 93.000000 +[Apr 27, 2026 - 14:51:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 218 : avg iteration time 52.88 seconds +[Apr 27, 2026 - 14:51:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 218 : data loading time 0.00 +[Apr 27, 2026 - 14:51:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 218 : avg forward pass time 14.35 +[Apr 27, 2026 - 14:51:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 218 : backward pass time 11.62 +[Apr 27, 2026 - 14:51:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 218 : optimizer step time 0.99 +[Apr 27, 2026 - 14:51:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 218-------------------- +[Apr 27, 2026 - 14:51:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0146 iter count: 1.0 +[Apr 27, 2026 - 14:51:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0146 iter count: 1.0 +[Apr 27, 2026 - 14:51:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 14:51:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 14:52:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077579 43.160351 42.987087 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.899902 69.957031 69.761719 +util 97.750000 99.000000 96.000000 +[Apr 27, 2026 - 14:52:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 219 : avg iteration time 49.63 seconds +[Apr 27, 2026 - 14:52:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 219 : data loading time 0.00 +[Apr 27, 2026 - 14:52:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 219 : avg forward pass time 12.66 +[Apr 27, 2026 - 14:52:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 219 : backward pass time 11.54 +[Apr 27, 2026 - 14:52:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 219 : optimizer step time 0.98 +[Apr 27, 2026 - 14:52:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 219-------------------- +[Apr 27, 2026 - 14:52:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0146 iter count: 1.0 +[Apr 27, 2026 - 14:52:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0146 iter count: 1.0 +[Apr 27, 2026 - 14:52:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 14:52:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 14:55:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077600 43.159397 42.987068 +peak_gpu_mem_gb 131.581049 131.620712 131.567828 +peak_gpu_mem_reserved_gb 135.567383 135.638672 135.517578 +util 76.250000 95.000000 64.000000 +[Apr 27, 2026 - 14:55:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 220 : avg iteration time 175.88 seconds +[Apr 27, 2026 - 14:55:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 220 : data loading time 0.00 +[Apr 27, 2026 - 14:55:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 220 : avg forward pass time 79.01 +[Apr 27, 2026 - 14:55:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 220 : backward pass time 9.08 +[Apr 27, 2026 - 14:55:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 220 : optimizer step time 0.34 +[Apr 27, 2026 - 14:55:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 220-------------------- +[Apr 27, 2026 - 14:55:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 348250.8135 iter count: 1.0 +[Apr 27, 2026 - 14:55:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6560 iter count: 1.0 +[Apr 27, 2026 - 14:55:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 348250.8135iter count: 1.0 +[Apr 27, 2026 - 14:55:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 14:55:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.8311iter count: 1.0 +[Apr 27, 2026 - 14:55:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.3924 iter count: 1.0 +[Apr 27, 2026 - 14:55:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.0359 iter count: 1.0 +[Apr 27, 2026 - 14:55:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 543077.8125iter count: 1.0 +[Apr 27, 2026 - 14:55:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 16.1952iter count: 1.0 +[Apr 27, 2026 - 14:55:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 9547999.0000iter count: 1.0 +[Apr 27, 2026 - 14:55:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6560iter count: 1.0 +[Apr 27, 2026 - 14:55:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.8911 iter count: 1.0 +[Apr 27, 2026 - 14:55:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 17714.5352iter count: 1.0 +[Apr 27, 2026 - 14:55:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 7.3371 iter count: 1.0 +[Apr 27, 2026 - 14:55:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 2359980.2500iter count: 1.0 +[Apr 27, 2026 - 14:55:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.0018 iter count: 1.0 +[Apr 27, 2026 - 14:55:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 404.8888iter count: 1.0 +[Apr 27, 2026 - 14:55:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.2376 iter count: 1.0 +[Apr 27, 2026 - 14:55:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 261786.0000iter count: 1.0 +[Apr 27, 2026 - 14:55:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0161 iter count: 1.0 +[Apr 27, 2026 - 14:55:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 14:55:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 14:55:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077611 43.159443 42.987064 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.923828 70.074219 69.648438 +util 97.250000 99.000000 95.000000 +[Apr 27, 2026 - 14:55:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 221 : avg iteration time 49.91 seconds +[Apr 27, 2026 - 14:55:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 221 : data loading time 0.00 +[Apr 27, 2026 - 14:55:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 221 : avg forward pass time 12.22 +[Apr 27, 2026 - 14:55:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 221 : backward pass time 12.84 +[Apr 27, 2026 - 14:55:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 221 : optimizer step time 1.07 +[Apr 27, 2026 - 14:55:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 221-------------------- +[Apr 27, 2026 - 14:55:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0142 iter count: 1.0 +[Apr 27, 2026 - 14:55:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0142 iter count: 1.0 +[Apr 27, 2026 - 14:55:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 14:55:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 14:56:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077605 43.159397 42.987087 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.958008 70.019531 69.898438 +util 96.750000 99.000000 94.000000 +[Apr 27, 2026 - 14:56:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 222 : avg iteration time 51.79 seconds +[Apr 27, 2026 - 14:56:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 222 : data loading time 0.00 +[Apr 27, 2026 - 14:56:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 222 : avg forward pass time 13.82 +[Apr 27, 2026 - 14:56:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 222 : backward pass time 11.57 +[Apr 27, 2026 - 14:56:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 222 : optimizer step time 0.98 +[Apr 27, 2026 - 14:56:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 222-------------------- +[Apr 27, 2026 - 14:56:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0156 iter count: 1.0 +[Apr 27, 2026 - 14:56:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0156 iter count: 1.0 +[Apr 27, 2026 - 14:56:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 14:56:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 14:57:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077606 43.159420 42.987064 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.821289 69.976562 69.550781 +util 98.250000 100.000000 93.000000 +[Apr 27, 2026 - 14:57:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 223 : avg iteration time 53.46 seconds +[Apr 27, 2026 - 14:57:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 223 : data loading time 0.00 +[Apr 27, 2026 - 14:57:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 223 : avg forward pass time 14.06 +[Apr 27, 2026 - 14:57:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 223 : backward pass time 11.76 +[Apr 27, 2026 - 14:57:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 223 : optimizer step time 1.51 +[Apr 27, 2026 - 14:57:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 223-------------------- +[Apr 27, 2026 - 14:57:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0145 iter count: 1.0 +[Apr 27, 2026 - 14:57:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0145 iter count: 1.0 +[Apr 27, 2026 - 14:57:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 14:57:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 14:58:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077607 43.159420 42.987064 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.831055 69.976562 69.570312 +util 98.000000 100.000000 94.000000 +[Apr 27, 2026 - 14:58:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 224 : avg iteration time 48.01 seconds +[Apr 27, 2026 - 14:58:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 224 : data loading time 0.00 +[Apr 27, 2026 - 14:58:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 224 : avg forward pass time 11.14 +[Apr 27, 2026 - 14:58:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 224 : backward pass time 13.01 +[Apr 27, 2026 - 14:58:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 224 : optimizer step time 1.16 +[Apr 27, 2026 - 14:58:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 224-------------------- +[Apr 27, 2026 - 14:58:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0140 iter count: 1.0 +[Apr 27, 2026 - 14:58:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0140 iter count: 1.0 +[Apr 27, 2026 - 14:58:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 14:58:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 15:01:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077677 43.159401 42.987087 +peak_gpu_mem_gb 131.581049 131.620712 131.567828 +peak_gpu_mem_reserved_gb 135.526855 135.677734 135.437500 +util 66.250000 73.000000 61.000000 +[Apr 27, 2026 - 15:01:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 225 : avg iteration time 180.90 seconds +[Apr 27, 2026 - 15:01:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 225 : data loading time 0.00 +[Apr 27, 2026 - 15:01:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 225 : avg forward pass time 81.28 +[Apr 27, 2026 - 15:01:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 225 : backward pass time 8.44 +[Apr 27, 2026 - 15:01:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 225 : optimizer step time 0.34 +[Apr 27, 2026 - 15:01:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 225-------------------- +[Apr 27, 2026 - 15:01:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 135286.4199 iter count: 1.0 +[Apr 27, 2026 - 15:01:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5991 iter count: 1.0 +[Apr 27, 2026 - 15:01:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 135286.4199iter count: 1.0 +[Apr 27, 2026 - 15:01:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:01:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.6809iter count: 1.0 +[Apr 27, 2026 - 15:01:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.4127 iter count: 1.0 +[Apr 27, 2026 - 15:01:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.5650 iter count: 1.0 +[Apr 27, 2026 - 15:01:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 239811.5469iter count: 1.0 +[Apr 27, 2026 - 15:01:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 16.8691iter count: 1.0 +[Apr 27, 2026 - 15:01:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 3722587.5000iter count: 1.0 +[Apr 27, 2026 - 15:01:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5991iter count: 1.0 +[Apr 27, 2026 - 15:01:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 1.4127 iter count: 1.0 +[Apr 27, 2026 - 15:01:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 16.8691 iter count: 1.0 +[Apr 27, 2026 - 15:01:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.9660 iter count: 1.0 +[Apr 27, 2026 - 15:01:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 20577.1445iter count: 1.0 +[Apr 27, 2026 - 15:01:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.3469 iter count: 1.0 +[Apr 27, 2026 - 15:01:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 5965.8320iter count: 1.0 +[Apr 27, 2026 - 15:01:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.8235 iter count: 1.0 +[Apr 27, 2026 - 15:01:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 15476.3457iter count: 1.0 +[Apr 27, 2026 - 15:01:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0154 iter count: 1.0 +[Apr 27, 2026 - 15:01:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:01:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 15:02:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077730 43.159424 42.987064 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.846191 69.923828 69.800781 +util 96.250000 99.000000 93.000000 +[Apr 27, 2026 - 15:02:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 226 : avg iteration time 54.02 seconds +[Apr 27, 2026 - 15:02:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 226 : data loading time 0.00 +[Apr 27, 2026 - 15:02:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 226 : avg forward pass time 14.84 +[Apr 27, 2026 - 15:02:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 226 : backward pass time 11.56 +[Apr 27, 2026 - 15:02:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 226 : optimizer step time 0.99 +[Apr 27, 2026 - 15:02:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 226-------------------- +[Apr 27, 2026 - 15:02:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0162 iter count: 1.0 +[Apr 27, 2026 - 15:02:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0162 iter count: 1.0 +[Apr 27, 2026 - 15:02:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:02:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 15:03:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077725 43.159405 42.987064 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.924316 70.041016 69.820312 +util 97.000000 100.000000 92.000000 +[Apr 27, 2026 - 15:03:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 227 : avg iteration time 53.78 seconds +[Apr 27, 2026 - 15:03:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 227 : data loading time 0.00 +[Apr 27, 2026 - 15:03:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 227 : avg forward pass time 14.20 +[Apr 27, 2026 - 15:03:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 227 : backward pass time 12.02 +[Apr 27, 2026 - 15:03:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 227 : optimizer step time 1.55 +[Apr 27, 2026 - 15:03:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 227-------------------- +[Apr 27, 2026 - 15:03:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0169 iter count: 1.0 +[Apr 27, 2026 - 15:03:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0169 iter count: 1.0 +[Apr 27, 2026 - 15:03:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:03:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 15:04:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077724 43.159401 42.987064 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.929199 69.957031 69.859375 +util 95.500000 100.000000 91.000000 +[Apr 27, 2026 - 15:04:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 228 : avg iteration time 48.03 seconds +[Apr 27, 2026 - 15:04:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 228 : data loading time 0.00 +[Apr 27, 2026 - 15:04:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 228 : avg forward pass time 11.11 +[Apr 27, 2026 - 15:04:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 228 : backward pass time 13.06 +[Apr 27, 2026 - 15:04:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 228 : optimizer step time 1.23 +[Apr 27, 2026 - 15:04:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 228-------------------- +[Apr 27, 2026 - 15:04:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0151 iter count: 1.0 +[Apr 27, 2026 - 15:04:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0151 iter count: 1.0 +[Apr 27, 2026 - 15:04:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:04:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 15:05:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077493 43.159416 42.987064 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.924316 69.937500 69.917969 +util 96.500000 99.000000 92.000000 +[Apr 27, 2026 - 15:05:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 229 : avg iteration time 52.83 seconds +[Apr 27, 2026 - 15:05:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 229 : data loading time 0.00 +[Apr 27, 2026 - 15:05:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 229 : avg forward pass time 14.37 +[Apr 27, 2026 - 15:05:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 229 : backward pass time 11.55 +[Apr 27, 2026 - 15:05:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 229 : optimizer step time 1.01 +[Apr 27, 2026 - 15:05:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 229-------------------- +[Apr 27, 2026 - 15:05:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0142 iter count: 1.0 +[Apr 27, 2026 - 15:05:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0142 iter count: 1.0 +[Apr 27, 2026 - 15:05:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:05:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 15:07:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077178 43.159401 42.987064 +peak_gpu_mem_gb 131.581049 131.620712 131.567828 +peak_gpu_mem_reserved_gb 135.537598 135.560547 135.476562 +util 68.250000 73.000000 61.000000 +[Apr 27, 2026 - 15:07:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 230 : avg iteration time 173.45 seconds +[Apr 27, 2026 - 15:07:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 230 : data loading time 0.00 +[Apr 27, 2026 - 15:07:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 230 : avg forward pass time 78.21 +[Apr 27, 2026 - 15:07:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 230 : backward pass time 8.38 +[Apr 27, 2026 - 15:07:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 230 : optimizer step time 0.34 +[Apr 27, 2026 - 15:07:54 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 230-------------------- +[Apr 27, 2026 - 15:07:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 159503.3384 iter count: 1.0 +[Apr 27, 2026 - 15:07:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6727 iter count: 1.0 +[Apr 27, 2026 - 15:07:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 159503.3384iter count: 1.0 +[Apr 27, 2026 - 15:07:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:07:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.4087iter count: 1.0 +[Apr 27, 2026 - 15:07:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.5384 iter count: 1.0 +[Apr 27, 2026 - 15:07:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.6329 iter count: 1.0 +[Apr 27, 2026 - 15:07:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 189097.6250iter count: 1.0 +[Apr 27, 2026 - 15:07:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 21.6892iter count: 1.0 +[Apr 27, 2026 - 15:07:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 4264411.5000iter count: 1.0 +[Apr 27, 2026 - 15:07:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6727iter count: 1.0 +[Apr 27, 2026 - 15:07:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.8669 iter count: 1.0 +[Apr 27, 2026 - 15:07:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 2284.3464iter count: 1.0 +[Apr 27, 2026 - 15:07:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.4194 iter count: 1.0 +[Apr 27, 2026 - 15:07:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 50963.7148iter count: 1.0 +[Apr 27, 2026 - 15:07:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.3007 iter count: 1.0 +[Apr 27, 2026 - 15:07:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 736.1506iter count: 1.0 +[Apr 27, 2026 - 15:07:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.4242 iter count: 1.0 +[Apr 27, 2026 - 15:07:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 6963.7183iter count: 1.0 +[Apr 27, 2026 - 15:07:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0140 iter count: 1.0 +[Apr 27, 2026 - 15:07:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:07:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 15:08:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077182 43.159420 42.987064 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.899414 69.937500 69.859375 +util 97.750000 100.000000 95.000000 +[Apr 27, 2026 - 15:08:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 231 : avg iteration time 55.12 seconds +[Apr 27, 2026 - 15:08:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 231 : data loading time 0.00 +[Apr 27, 2026 - 15:08:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 231 : avg forward pass time 14.85 +[Apr 27, 2026 - 15:08:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 231 : backward pass time 11.60 +[Apr 27, 2026 - 15:08:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 231 : optimizer step time 0.97 +[Apr 27, 2026 - 15:08:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 231-------------------- +[Apr 27, 2026 - 15:08:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0152 iter count: 1.0 +[Apr 27, 2026 - 15:08:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0152 iter count: 1.0 +[Apr 27, 2026 - 15:08:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:08:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 15:09:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077545 43.159401 42.987064 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.997070 70.035156 69.941406 +util 98.750000 100.000000 95.000000 +[Apr 27, 2026 - 15:09:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 232 : avg iteration time 50.98 seconds +[Apr 27, 2026 - 15:09:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 232 : data loading time 0.00 +[Apr 27, 2026 - 15:09:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 232 : avg forward pass time 12.97 +[Apr 27, 2026 - 15:09:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 232 : backward pass time 11.92 +[Apr 27, 2026 - 15:09:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 232 : optimizer step time 1.53 +[Apr 27, 2026 - 15:09:40 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 232-------------------- +[Apr 27, 2026 - 15:09:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0132 iter count: 1.0 +[Apr 27, 2026 - 15:09:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0132 iter count: 1.0 +[Apr 27, 2026 - 15:09:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:09:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 15:10:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077611 43.159401 42.987080 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.928711 69.980469 69.878906 +util 96.500000 99.000000 93.000000 +[Apr 27, 2026 - 15:10:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 233 : avg iteration time 48.00 seconds +[Apr 27, 2026 - 15:10:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 233 : data loading time 0.00 +[Apr 27, 2026 - 15:10:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 233 : avg forward pass time 11.12 +[Apr 27, 2026 - 15:10:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 233 : backward pass time 13.04 +[Apr 27, 2026 - 15:10:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 233 : optimizer step time 1.16 +[Apr 27, 2026 - 15:10:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 233-------------------- +[Apr 27, 2026 - 15:10:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0125 iter count: 1.0 +[Apr 27, 2026 - 15:10:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0125 iter count: 1.0 +[Apr 27, 2026 - 15:10:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:10:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 15:11:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077601 43.159401 42.987064 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.987305 69.996094 69.976562 +util 98.000000 99.000000 95.000000 +[Apr 27, 2026 - 15:11:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 234 : avg iteration time 50.14 seconds +[Apr 27, 2026 - 15:11:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 234 : data loading time 0.00 +[Apr 27, 2026 - 15:11:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 234 : avg forward pass time 12.98 +[Apr 27, 2026 - 15:11:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 234 : backward pass time 11.61 +[Apr 27, 2026 - 15:11:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 234 : optimizer step time 1.00 +[Apr 27, 2026 - 15:11:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 234-------------------- +[Apr 27, 2026 - 15:11:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0137 iter count: 1.0 +[Apr 27, 2026 - 15:11:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0137 iter count: 1.0 +[Apr 27, 2026 - 15:11:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:11:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 15:14:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077065 43.159401 42.987064 +peak_gpu_mem_gb 131.581049 131.620712 131.567828 +peak_gpu_mem_reserved_gb 135.532227 135.554688 135.478516 +util 68.750000 76.000000 64.000000 +[Apr 27, 2026 - 15:14:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 235 : avg iteration time 172.86 seconds +[Apr 27, 2026 - 15:14:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 235 : data loading time 0.00 +[Apr 27, 2026 - 15:14:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 235 : avg forward pass time 77.92 +[Apr 27, 2026 - 15:14:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 235 : backward pass time 8.38 +[Apr 27, 2026 - 15:14:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 235 : optimizer step time 0.34 +[Apr 27, 2026 - 15:14:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 235-------------------- +[Apr 27, 2026 - 15:14:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 445974.0366 iter count: 1.0 +[Apr 27, 2026 - 15:14:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6609 iter count: 1.0 +[Apr 27, 2026 - 15:14:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 445974.0366iter count: 1.0 +[Apr 27, 2026 - 15:14:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:14:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.7267iter count: 1.0 +[Apr 27, 2026 - 15:14:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.5020 iter count: 1.0 +[Apr 27, 2026 - 15:14:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.0114 iter count: 1.0 +[Apr 27, 2026 - 15:14:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 625039.0000iter count: 1.0 +[Apr 27, 2026 - 15:14:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 2.7292 iter count: 1.0 +[Apr 27, 2026 - 15:14:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 9091323.0000iter count: 1.0 +[Apr 27, 2026 - 15:14:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6609iter count: 1.0 +[Apr 27, 2026 - 15:14:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 6.5264 iter count: 1.0 +[Apr 27, 2026 - 15:14:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 466401.0625iter count: 1.0 +[Apr 27, 2026 - 15:14:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.2386 iter count: 1.0 +[Apr 27, 2026 - 15:14:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 4804.0693iter count: 1.0 +[Apr 27, 2026 - 15:14:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 2.9238 iter count: 1.0 +[Apr 27, 2026 - 15:14:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 346.4314iter count: 1.0 +[Apr 27, 2026 - 15:14:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 2.9036 iter count: 1.0 +[Apr 27, 2026 - 15:14:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 332.6557iter count: 1.0 +[Apr 27, 2026 - 15:14:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0122 iter count: 1.0 +[Apr 27, 2026 - 15:14:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:14:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 15:15:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077127 43.159401 42.987064 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.051270 70.099609 69.957031 +util 97.250000 100.000000 94.000000 +[Apr 27, 2026 - 15:15:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 236 : avg iteration time 49.21 seconds +[Apr 27, 2026 - 15:15:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 236 : data loading time 0.00 +[Apr 27, 2026 - 15:15:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 236 : avg forward pass time 12.21 +[Apr 27, 2026 - 15:15:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 236 : backward pass time 11.54 +[Apr 27, 2026 - 15:15:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 236 : optimizer step time 0.99 +[Apr 27, 2026 - 15:15:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 236-------------------- +[Apr 27, 2026 - 15:15:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0108 iter count: 1.0 +[Apr 27, 2026 - 15:15:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[Apr 27, 2026 - 15:15:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:15:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 15:15:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077500 43.159401 42.987064 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.943359 70.058594 69.878906 +util 97.250000 100.000000 94.000000 +[Apr 27, 2026 - 15:15:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 237 : avg iteration time 48.00 seconds +[Apr 27, 2026 - 15:15:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 237 : data loading time 0.00 +[Apr 27, 2026 - 15:15:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 237 : avg forward pass time 11.18 +[Apr 27, 2026 - 15:15:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 237 : backward pass time 11.56 +[Apr 27, 2026 - 15:15:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 237 : optimizer step time 0.98 +[Apr 27, 2026 - 15:15:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 237-------------------- +[Apr 27, 2026 - 15:15:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0118 iter count: 1.0 +[Apr 27, 2026 - 15:15:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0118 iter count: 1.0 +[Apr 27, 2026 - 15:15:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:15:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 15:16:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077501 43.159401 42.987068 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.982422 70.117188 69.937500 +util 99.000000 100.000000 97.000000 +[Apr 27, 2026 - 15:16:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 238 : avg iteration time 54.02 seconds +[Apr 27, 2026 - 15:16:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 238 : data loading time 0.00 +[Apr 27, 2026 - 15:16:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 238 : avg forward pass time 14.27 +[Apr 27, 2026 - 15:16:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 238 : backward pass time 12.29 +[Apr 27, 2026 - 15:16:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 238 : optimizer step time 1.57 +[Apr 27, 2026 - 15:16:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 238-------------------- +[Apr 27, 2026 - 15:16:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0129 iter count: 1.0 +[Apr 27, 2026 - 15:16:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0129 iter count: 1.0 +[Apr 27, 2026 - 15:16:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:16:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 15:17:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077466 43.159401 42.987068 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.031250 70.097656 69.878906 +util 97.500000 99.000000 95.000000 +[Apr 27, 2026 - 15:17:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 239 : avg iteration time 53.00 seconds +[Apr 27, 2026 - 15:17:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 239 : data loading time 0.00 +[Apr 27, 2026 - 15:17:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 239 : avg forward pass time 14.39 +[Apr 27, 2026 - 15:17:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 239 : backward pass time 11.65 +[Apr 27, 2026 - 15:17:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 239 : optimizer step time 0.99 +[Apr 27, 2026 - 15:17:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 239-------------------- +[Apr 27, 2026 - 15:17:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0122 iter count: 1.0 +[Apr 27, 2026 - 15:17:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0122 iter count: 1.0 +[Apr 27, 2026 - 15:17:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:17:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 15:20:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077317 43.159401 42.987068 +peak_gpu_mem_gb 131.581049 131.620712 131.567828 +peak_gpu_mem_reserved_gb 135.556641 135.619141 135.535156 +util 68.000000 75.000000 63.000000 +[Apr 27, 2026 - 15:20:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 240 : avg iteration time 175.98 seconds +[Apr 27, 2026 - 15:20:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 240 : data loading time 0.00 +[Apr 27, 2026 - 15:20:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 240 : avg forward pass time 79.47 +[Apr 27, 2026 - 15:20:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 240 : backward pass time 8.39 +[Apr 27, 2026 - 15:20:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 240 : optimizer step time 0.34 +[Apr 27, 2026 - 15:20:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 240-------------------- +[Apr 27, 2026 - 15:20:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 726209.5559 iter count: 1.0 +[Apr 27, 2026 - 15:20:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7020 iter count: 1.0 +[Apr 27, 2026 - 15:20:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 726209.5559iter count: 1.0 +[Apr 27, 2026 - 15:20:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:20:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.5550iter count: 1.0 +[Apr 27, 2026 - 15:20:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.1070 iter count: 1.0 +[Apr 27, 2026 - 15:20:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.3422 iter count: 1.0 +[Apr 27, 2026 - 15:20:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 963205.1250iter count: 1.0 +[Apr 27, 2026 - 15:20:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 9.1514 iter count: 1.0 +[Apr 27, 2026 - 15:20:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 17618156.0000iter count: 1.0 +[Apr 27, 2026 - 15:20:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7020iter count: 1.0 +[Apr 27, 2026 - 15:20:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.9680 iter count: 1.0 +[Apr 27, 2026 - 15:20:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 20660.6953iter count: 1.0 +[Apr 27, 2026 - 15:20:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.1056 iter count: 1.0 +[Apr 27, 2026 - 15:20:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 3681.7573iter count: 1.0 +[Apr 27, 2026 - 15:20:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.2196 iter count: 1.0 +[Apr 27, 2026 - 15:20:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 4624.8066iter count: 1.0 +[Apr 27, 2026 - 15:20:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.4476 iter count: 1.0 +[Apr 27, 2026 - 15:20:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 7296.8281iter count: 1.0 +[Apr 27, 2026 - 15:20:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[Apr 27, 2026 - 15:20:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:20:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 15:21:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077316 43.159401 42.987068 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.943359 70.039062 69.859375 +util 96.250000 100.000000 91.000000 +[Apr 27, 2026 - 15:21:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 241 : avg iteration time 52.44 seconds +[Apr 27, 2026 - 15:21:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 241 : data loading time 0.00 +[Apr 27, 2026 - 15:21:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 241 : avg forward pass time 13.38 +[Apr 27, 2026 - 15:21:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 241 : backward pass time 11.56 +[Apr 27, 2026 - 15:21:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 241 : optimizer step time 0.97 +[Apr 27, 2026 - 15:21:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 241-------------------- +[Apr 27, 2026 - 15:21:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0139 iter count: 1.0 +[Apr 27, 2026 - 15:21:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0139 iter count: 1.0 +[Apr 27, 2026 - 15:21:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:21:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 15:22:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077499 43.159401 42.987068 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.011719 70.074219 69.980469 +util 96.000000 99.000000 90.000000 +[Apr 27, 2026 - 15:22:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 242 : avg iteration time 47.72 seconds +[Apr 27, 2026 - 15:22:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 242 : data loading time 0.00 +[Apr 27, 2026 - 15:22:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 242 : avg forward pass time 11.23 +[Apr 27, 2026 - 15:22:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 242 : backward pass time 11.53 +[Apr 27, 2026 - 15:22:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 242 : optimizer step time 0.98 +[Apr 27, 2026 - 15:22:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 242-------------------- +[Apr 27, 2026 - 15:22:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0114 iter count: 1.0 +[Apr 27, 2026 - 15:22:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[Apr 27, 2026 - 15:22:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:22:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 15:23:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077562 43.159401 42.987068 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.011719 70.074219 69.980469 +util 99.250000 100.000000 97.000000 +[Apr 27, 2026 - 15:23:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 243 : avg iteration time 51.88 seconds +[Apr 27, 2026 - 15:23:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 243 : data loading time 0.00 +[Apr 27, 2026 - 15:23:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 243 : avg forward pass time 13.05 +[Apr 27, 2026 - 15:23:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 243 : backward pass time 12.64 +[Apr 27, 2026 - 15:23:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 243 : optimizer step time 1.60 +[Apr 27, 2026 - 15:23:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 243-------------------- +[Apr 27, 2026 - 15:23:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0112 iter count: 1.0 +[Apr 27, 2026 - 15:23:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[Apr 27, 2026 - 15:23:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:23:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 15:23:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077485 43.159401 42.987068 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.894531 69.980469 69.839844 +util 97.500000 100.000000 93.000000 +[Apr 27, 2026 - 15:23:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 244 : avg iteration time 48.32 seconds +[Apr 27, 2026 - 15:23:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 244 : data loading time 0.00 +[Apr 27, 2026 - 15:23:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 244 : avg forward pass time 11.24 +[Apr 27, 2026 - 15:23:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 244 : backward pass time 13.05 +[Apr 27, 2026 - 15:23:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 244 : optimizer step time 1.22 +[Apr 27, 2026 - 15:23:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 244-------------------- +[Apr 27, 2026 - 15:23:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0125 iter count: 1.0 +[Apr 27, 2026 - 15:23:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0125 iter count: 1.0 +[Apr 27, 2026 - 15:23:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:23:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 15:26:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077234 43.159401 42.987068 +peak_gpu_mem_gb 131.581049 131.620712 131.567828 +peak_gpu_mem_reserved_gb 135.576172 135.638672 135.517578 +util 69.500000 74.000000 67.000000 +[Apr 27, 2026 - 15:26:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 245 : avg iteration time 170.68 seconds +[Apr 27, 2026 - 15:26:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 245 : data loading time 0.00 +[Apr 27, 2026 - 15:26:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 245 : avg forward pass time 76.74 +[Apr 27, 2026 - 15:26:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 245 : backward pass time 8.39 +[Apr 27, 2026 - 15:26:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 245 : optimizer step time 0.34 +[Apr 27, 2026 - 15:26:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 245-------------------- +[Apr 27, 2026 - 15:26:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 54259.4956 iter count: 1.0 +[Apr 27, 2026 - 15:26:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6880 iter count: 1.0 +[Apr 27, 2026 - 15:26:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 54259.4956iter count: 1.0 +[Apr 27, 2026 - 15:26:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:26:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.3080iter count: 1.0 +[Apr 27, 2026 - 15:26:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.6875 iter count: 1.0 +[Apr 27, 2026 - 15:26:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.1709 iter count: 1.0 +[Apr 27, 2026 - 15:26:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 83947.9219iter count: 1.0 +[Apr 27, 2026 - 15:26:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 29.2257iter count: 1.0 +[Apr 27, 2026 - 15:26:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 1692788.1250iter count: 1.0 +[Apr 27, 2026 - 15:26:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6880iter count: 1.0 +[Apr 27, 2026 - 15:26:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 2.2424 iter count: 1.0 +[Apr 27, 2026 - 15:26:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 88.6609 iter count: 1.0 +[Apr 27, 2026 - 15:26:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.1947 iter count: 1.0 +[Apr 27, 2026 - 15:26:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 4400.2627iter count: 1.0 +[Apr 27, 2026 - 15:26:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 7.1709 iter count: 1.0 +[Apr 27, 2026 - 15:26:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 1692788.1250iter count: 1.0 +[Apr 27, 2026 - 15:26:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.6317 iter count: 1.0 +[Apr 27, 2026 - 15:26:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 1426.9603iter count: 1.0 +[Apr 27, 2026 - 15:26:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[Apr 27, 2026 - 15:26:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:26:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 15:27:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077355 43.159401 42.987072 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.914062 69.941406 69.898438 +util 98.000000 100.000000 94.000000 +[Apr 27, 2026 - 15:27:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 246 : avg iteration time 51.70 seconds +[Apr 27, 2026 - 15:27:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 246 : data loading time 0.00 +[Apr 27, 2026 - 15:27:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 246 : avg forward pass time 13.50 +[Apr 27, 2026 - 15:27:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 246 : backward pass time 11.56 +[Apr 27, 2026 - 15:27:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 246 : optimizer step time 0.98 +[Apr 27, 2026 - 15:27:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 246-------------------- +[Apr 27, 2026 - 15:27:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0150 iter count: 1.0 +[Apr 27, 2026 - 15:27:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0150 iter count: 1.0 +[Apr 27, 2026 - 15:27:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:27:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 15:28:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077356 43.159401 42.987068 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.992188 70.132812 69.839844 +util 97.750000 99.000000 94.000000 +[Apr 27, 2026 - 15:28:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 247 : avg iteration time 50.53 seconds +[Apr 27, 2026 - 15:28:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 247 : data loading time 0.00 +[Apr 27, 2026 - 15:28:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 247 : avg forward pass time 12.61 +[Apr 27, 2026 - 15:28:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 247 : backward pass time 11.55 +[Apr 27, 2026 - 15:28:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 247 : optimizer step time 1.01 +[Apr 27, 2026 - 15:28:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 247-------------------- +[Apr 27, 2026 - 15:28:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0154 iter count: 1.0 +[Apr 27, 2026 - 15:28:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0154 iter count: 1.0 +[Apr 27, 2026 - 15:28:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:28:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 15:29:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077360 43.159401 42.987068 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.962891 70.019531 69.800781 +util 96.500000 99.000000 91.000000 +[Apr 27, 2026 - 15:29:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 248 : avg iteration time 53.29 seconds +[Apr 27, 2026 - 15:29:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 248 : data loading time 0.00 +[Apr 27, 2026 - 15:29:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 248 : avg forward pass time 14.22 +[Apr 27, 2026 - 15:29:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 248 : backward pass time 12.28 +[Apr 27, 2026 - 15:29:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 248 : optimizer step time 1.00 +[Apr 27, 2026 - 15:29:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 248-------------------- +[Apr 27, 2026 - 15:29:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0144 iter count: 1.0 +[Apr 27, 2026 - 15:29:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0144 iter count: 1.0 +[Apr 27, 2026 - 15:29:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:29:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 15:30:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077549 43.159405 42.987068 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.918945 70.015625 69.800781 +util 97.250000 99.000000 93.000000 +[Apr 27, 2026 - 15:30:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 249 : avg iteration time 49.50 seconds +[Apr 27, 2026 - 15:30:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 249 : data loading time 0.00 +[Apr 27, 2026 - 15:30:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 249 : avg forward pass time 12.66 +[Apr 27, 2026 - 15:30:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 249 : backward pass time 11.55 +[Apr 27, 2026 - 15:30:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 249 : optimizer step time 0.97 +[Apr 27, 2026 - 15:30:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 249-------------------- +[Apr 27, 2026 - 15:30:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0125 iter count: 1.0 +[Apr 27, 2026 - 15:30:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0125 iter count: 1.0 +[Apr 27, 2026 - 15:30:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:30:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 15:33:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077374 43.159401 42.987068 +peak_gpu_mem_gb 131.581049 131.620712 131.567828 +peak_gpu_mem_reserved_gb 135.566895 135.658203 135.535156 +util 69.500000 77.000000 62.000000 +[Apr 27, 2026 - 15:33:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 250 : avg iteration time 181.70 seconds +[Apr 27, 2026 - 15:33:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 250 : data loading time 0.00 +[Apr 27, 2026 - 15:33:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 250 : avg forward pass time 82.30 +[Apr 27, 2026 - 15:33:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 250 : backward pass time 8.45 +[Apr 27, 2026 - 15:33:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 250 : optimizer step time 0.34 +[Apr 27, 2026 - 15:33:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 250-------------------- +[Apr 27, 2026 - 15:33:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 362574.5273 iter count: 1.0 +[Apr 27, 2026 - 15:33:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6233 iter count: 1.0 +[Apr 27, 2026 - 15:33:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 362574.5273iter count: 1.0 +[Apr 27, 2026 - 15:33:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:33:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.5921iter count: 1.0 +[Apr 27, 2026 - 15:33:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.8327 iter count: 1.0 +[Apr 27, 2026 - 15:33:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.0324 iter count: 1.0 +[Apr 27, 2026 - 15:33:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 453608.5625iter count: 1.0 +[Apr 27, 2026 - 15:33:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 5.2875 iter count: 1.0 +[Apr 27, 2026 - 15:33:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 9481415.0000iter count: 1.0 +[Apr 27, 2026 - 15:33:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6233iter count: 1.0 +[Apr 27, 2026 - 15:33:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.8930 iter count: 1.0 +[Apr 27, 2026 - 15:33:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 2406.7937iter count: 1.0 +[Apr 27, 2026 - 15:33:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.3476 iter count: 1.0 +[Apr 27, 2026 - 15:33:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 808.4446iter count: 1.0 +[Apr 27, 2026 - 15:33:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.8129 iter count: 1.0 +[Apr 27, 2026 - 15:33:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 2050.3521iter count: 1.0 +[Apr 27, 2026 - 15:33:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.0132 iter count: 1.0 +[Apr 27, 2026 - 15:33:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 22615.3984iter count: 1.0 +[Apr 27, 2026 - 15:33:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0150 iter count: 1.0 +[Apr 27, 2026 - 15:33:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:33:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 15:34:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077377 43.159401 42.987080 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.889648 69.941406 69.859375 +util 98.500000 100.000000 96.000000 +[Apr 27, 2026 - 15:34:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 251 : avg iteration time 54.66 seconds +[Apr 27, 2026 - 15:34:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 251 : data loading time 0.00 +[Apr 27, 2026 - 15:34:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 251 : avg forward pass time 14.69 +[Apr 27, 2026 - 15:34:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 251 : backward pass time 11.55 +[Apr 27, 2026 - 15:34:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 251 : optimizer step time 1.27 +[Apr 27, 2026 - 15:34:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 251-------------------- +[Apr 27, 2026 - 15:34:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0149 iter count: 1.0 +[Apr 27, 2026 - 15:34:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0149 iter count: 1.0 +[Apr 27, 2026 - 15:34:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:34:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 15:34:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077380 43.159401 42.987068 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.914062 69.957031 69.898438 +util 96.750000 100.000000 91.000000 +[Apr 27, 2026 - 15:34:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 252 : avg iteration time 54.27 seconds +[Apr 27, 2026 - 15:34:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 252 : data loading time 0.00 +[Apr 27, 2026 - 15:34:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 252 : avg forward pass time 14.26 +[Apr 27, 2026 - 15:34:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 252 : backward pass time 12.56 +[Apr 27, 2026 - 15:34:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 252 : optimizer step time 1.59 +[Apr 27, 2026 - 15:34:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 252-------------------- +[Apr 27, 2026 - 15:34:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0169 iter count: 1.0 +[Apr 27, 2026 - 15:34:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0169 iter count: 1.0 +[Apr 27, 2026 - 15:34:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:34:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 15:35:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077380 43.159401 42.987068 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.948242 69.976562 69.917969 +util 96.500000 99.000000 93.000000 +[Apr 27, 2026 - 15:35:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 253 : avg iteration time 50.51 seconds +[Apr 27, 2026 - 15:35:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 253 : data loading time 0.00 +[Apr 27, 2026 - 15:35:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 253 : avg forward pass time 12.60 +[Apr 27, 2026 - 15:35:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 253 : backward pass time 12.70 +[Apr 27, 2026 - 15:35:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 253 : optimizer step time 1.04 +[Apr 27, 2026 - 15:35:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 253-------------------- +[Apr 27, 2026 - 15:35:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0145 iter count: 1.0 +[Apr 27, 2026 - 15:35:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0145 iter count: 1.0 +[Apr 27, 2026 - 15:35:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:35:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 15:36:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077380 43.159401 42.987068 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.884766 69.941406 69.839844 +util 97.250000 99.000000 93.000000 +[Apr 27, 2026 - 15:36:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 254 : avg iteration time 49.06 seconds +[Apr 27, 2026 - 15:36:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 254 : data loading time 0.00 +[Apr 27, 2026 - 15:36:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 254 : avg forward pass time 12.48 +[Apr 27, 2026 - 15:36:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 254 : backward pass time 11.54 +[Apr 27, 2026 - 15:36:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 254 : optimizer step time 0.99 +[Apr 27, 2026 - 15:36:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 254-------------------- +[Apr 27, 2026 - 15:36:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0132 iter count: 1.0 +[Apr 27, 2026 - 15:36:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0132 iter count: 1.0 +[Apr 27, 2026 - 15:36:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:36:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 15:39:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077110 43.159401 42.987068 +peak_gpu_mem_gb 131.581049 131.620712 131.567828 +peak_gpu_mem_reserved_gb 135.552246 135.619141 135.437500 +util 71.250000 76.000000 64.000000 +[Apr 27, 2026 - 15:39:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 255 : avg iteration time 176.27 seconds +[Apr 27, 2026 - 15:39:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 255 : data loading time 0.00 +[Apr 27, 2026 - 15:39:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 255 : avg forward pass time 79.63 +[Apr 27, 2026 - 15:39:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 255 : backward pass time 8.40 +[Apr 27, 2026 - 15:39:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 255 : optimizer step time 0.34 +[Apr 27, 2026 - 15:39:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 255-------------------- +[Apr 27, 2026 - 15:39:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 99894.2656 iter count: 1.0 +[Apr 27, 2026 - 15:39:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6417 iter count: 1.0 +[Apr 27, 2026 - 15:39:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 99894.2656iter count: 1.0 +[Apr 27, 2026 - 15:39:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:39:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.2366iter count: 1.0 +[Apr 27, 2026 - 15:39:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.5366 iter count: 1.0 +[Apr 27, 2026 - 15:39:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.1674 iter count: 1.0 +[Apr 27, 2026 - 15:39:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 173549.5000iter count: 1.0 +[Apr 27, 2026 - 15:39:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 21.6114iter count: 1.0 +[Apr 27, 2026 - 15:39:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 1680843.8750iter count: 1.0 +[Apr 27, 2026 - 15:39:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6417iter count: 1.0 +[Apr 27, 2026 - 15:39:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 7.1046 iter count: 1.0 +[Apr 27, 2026 - 15:39:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 1482389.0000iter count: 1.0 +[Apr 27, 2026 - 15:39:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.2722 iter count: 1.0 +[Apr 27, 2026 - 15:39:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 695.2831iter count: 1.0 +[Apr 27, 2026 - 15:39:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.9061 iter count: 1.0 +[Apr 27, 2026 - 15:39:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 2470.4480iter count: 1.0 +[Apr 27, 2026 - 15:39:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.4042 iter count: 1.0 +[Apr 27, 2026 - 15:39:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 905.4617iter count: 1.0 +[Apr 27, 2026 - 15:39:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0143 iter count: 1.0 +[Apr 27, 2026 - 15:39:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:39:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 15:40:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077108 43.159401 42.987068 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.923828 70.097656 69.839844 +util 97.000000 100.000000 92.000000 +[Apr 27, 2026 - 15:40:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 256 : avg iteration time 52.17 seconds +[Apr 27, 2026 - 15:40:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 256 : data loading time 0.00 +[Apr 27, 2026 - 15:40:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 256 : avg forward pass time 13.31 +[Apr 27, 2026 - 15:40:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 256 : backward pass time 11.52 +[Apr 27, 2026 - 15:40:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 256 : optimizer step time 0.99 +[Apr 27, 2026 - 15:40:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 256-------------------- +[Apr 27, 2026 - 15:40:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0181 iter count: 1.0 +[Apr 27, 2026 - 15:40:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0181 iter count: 1.0 +[Apr 27, 2026 - 15:40:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:40:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 15:41:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077357 43.159401 42.987068 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.953125 70.097656 69.839844 +util 98.500000 100.000000 95.000000 +[Apr 27, 2026 - 15:41:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 257 : avg iteration time 48.69 seconds +[Apr 27, 2026 - 15:41:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 257 : data loading time 0.00 +[Apr 27, 2026 - 15:41:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 257 : avg forward pass time 11.63 +[Apr 27, 2026 - 15:41:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 257 : backward pass time 12.30 +[Apr 27, 2026 - 15:41:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 257 : optimizer step time 1.59 +[Apr 27, 2026 - 15:41:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 257-------------------- +[Apr 27, 2026 - 15:41:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0148 iter count: 1.0 +[Apr 27, 2026 - 15:41:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0148 iter count: 1.0 +[Apr 27, 2026 - 15:41:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:41:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 15:42:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077548 43.159401 42.987068 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.914062 69.960938 69.820312 +util 97.250000 99.000000 93.000000 +[Apr 27, 2026 - 15:42:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 258 : avg iteration time 53.06 seconds +[Apr 27, 2026 - 15:42:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 258 : data loading time 0.00 +[Apr 27, 2026 - 15:42:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 258 : avg forward pass time 14.38 +[Apr 27, 2026 - 15:42:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 258 : backward pass time 11.71 +[Apr 27, 2026 - 15:42:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 258 : optimizer step time 0.99 +[Apr 27, 2026 - 15:42:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 258-------------------- +[Apr 27, 2026 - 15:42:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0168 iter count: 1.0 +[Apr 27, 2026 - 15:42:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0168 iter count: 1.0 +[Apr 27, 2026 - 15:42:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:42:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 15:42:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077664 43.159401 42.987068 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.933594 70.019531 69.839844 +util 97.500000 100.000000 93.000000 +[Apr 27, 2026 - 15:42:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 259 : avg iteration time 50.06 seconds +[Apr 27, 2026 - 15:42:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 259 : data loading time 0.00 +[Apr 27, 2026 - 15:42:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 259 : avg forward pass time 12.70 +[Apr 27, 2026 - 15:42:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 259 : backward pass time 11.56 +[Apr 27, 2026 - 15:42:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 259 : optimizer step time 0.97 +[Apr 27, 2026 - 15:42:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 259-------------------- +[Apr 27, 2026 - 15:42:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0168 iter count: 1.0 +[Apr 27, 2026 - 15:42:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0168 iter count: 1.0 +[Apr 27, 2026 - 15:42:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:42:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 15:45:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077011 43.159401 42.987072 +peak_gpu_mem_gb 131.581049 131.620712 131.567828 +peak_gpu_mem_reserved_gb 135.567383 135.658203 135.498047 +util 69.750000 76.000000 63.000000 +[Apr 27, 2026 - 15:45:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 260 : avg iteration time 172.95 seconds +[Apr 27, 2026 - 15:45:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 260 : data loading time 0.00 +[Apr 27, 2026 - 15:45:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 260 : avg forward pass time 77.95 +[Apr 27, 2026 - 15:45:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 260 : backward pass time 8.39 +[Apr 27, 2026 - 15:45:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 260 : optimizer step time 0.34 +[Apr 27, 2026 - 15:45:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 260-------------------- +[Apr 27, 2026 - 15:45:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 141057.5730 iter count: 1.0 +[Apr 27, 2026 - 15:45:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6693 iter count: 1.0 +[Apr 27, 2026 - 15:45:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 141057.5730iter count: 1.0 +[Apr 27, 2026 - 15:45:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:45:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.1350iter count: 1.0 +[Apr 27, 2026 - 15:45:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.6532 iter count: 1.0 +[Apr 27, 2026 - 15:45:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.3937 iter count: 1.0 +[Apr 27, 2026 - 15:45:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 248599.8438iter count: 1.0 +[Apr 27, 2026 - 15:45:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 3.6930 iter count: 1.0 +[Apr 27, 2026 - 15:45:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 2643022.2500iter count: 1.0 +[Apr 27, 2026 - 15:45:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6693iter count: 1.0 +[Apr 27, 2026 - 15:45:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.4811 iter count: 1.0 +[Apr 27, 2026 - 15:45:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 57655.7891iter count: 1.0 +[Apr 27, 2026 - 15:45:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 0.6532 iter count: 1.0 +[Apr 27, 2026 - 15:45:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 3.6930 iter count: 1.0 +[Apr 27, 2026 - 15:45:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 2.4221 iter count: 1.0 +[Apr 27, 2026 - 15:45:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 127.0097iter count: 1.0 +[Apr 27, 2026 - 15:45:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 2.7585 iter count: 1.0 +[Apr 27, 2026 - 15:45:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 248.8802iter count: 1.0 +[Apr 27, 2026 - 15:45:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0166 iter count: 1.0 +[Apr 27, 2026 - 15:45:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:45:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 15:46:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077082 43.159401 42.987091 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.943848 70.001953 69.898438 +util 96.250000 100.000000 92.000000 +[Apr 27, 2026 - 15:46:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 261 : avg iteration time 49.73 seconds +[Apr 27, 2026 - 15:46:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 261 : data loading time 0.00 +[Apr 27, 2026 - 15:46:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 261 : avg forward pass time 12.03 +[Apr 27, 2026 - 15:46:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 261 : backward pass time 11.57 +[Apr 27, 2026 - 15:46:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 261 : optimizer step time 0.97 +[Apr 27, 2026 - 15:46:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 261-------------------- +[Apr 27, 2026 - 15:46:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0153 iter count: 1.0 +[Apr 27, 2026 - 15:46:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0153 iter count: 1.0 +[Apr 27, 2026 - 15:46:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:46:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 15:47:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077515 43.159401 42.987076 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.982910 70.041016 69.957031 +util 98.250000 100.000000 93.000000 +[Apr 27, 2026 - 15:47:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 262 : avg iteration time 53.87 seconds +[Apr 27, 2026 - 15:47:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 262 : data loading time 0.00 +[Apr 27, 2026 - 15:47:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 262 : avg forward pass time 14.25 +[Apr 27, 2026 - 15:47:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 262 : backward pass time 12.16 +[Apr 27, 2026 - 15:47:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 262 : optimizer step time 1.58 +[Apr 27, 2026 - 15:47:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 262-------------------- +[Apr 27, 2026 - 15:47:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0168 iter count: 1.0 +[Apr 27, 2026 - 15:47:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0168 iter count: 1.0 +[Apr 27, 2026 - 15:47:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:47:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 15:48:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077573 43.159401 42.987068 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.978027 70.060547 69.937500 +util 96.000000 99.000000 92.000000 +[Apr 27, 2026 - 15:48:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 263 : avg iteration time 48.13 seconds +[Apr 27, 2026 - 15:48:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 263 : data loading time 0.00 +[Apr 27, 2026 - 15:48:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 263 : avg forward pass time 11.21 +[Apr 27, 2026 - 15:48:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 263 : backward pass time 13.01 +[Apr 27, 2026 - 15:48:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 263 : optimizer step time 1.14 +[Apr 27, 2026 - 15:48:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 263-------------------- +[Apr 27, 2026 - 15:48:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0157 iter count: 1.0 +[Apr 27, 2026 - 15:48:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0157 iter count: 1.0 +[Apr 27, 2026 - 15:48:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:48:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 15:49:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077397 43.159401 42.987068 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.953613 70.021484 69.917969 +util 97.500000 99.000000 95.000000 +[Apr 27, 2026 - 15:49:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 264 : avg iteration time 50.04 seconds +[Apr 27, 2026 - 15:49:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 264 : data loading time 0.00 +[Apr 27, 2026 - 15:49:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 264 : avg forward pass time 12.94 +[Apr 27, 2026 - 15:49:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 264 : backward pass time 11.60 +[Apr 27, 2026 - 15:49:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 264 : optimizer step time 1.00 +[Apr 27, 2026 - 15:49:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 264-------------------- +[Apr 27, 2026 - 15:49:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0144 iter count: 1.0 +[Apr 27, 2026 - 15:49:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0144 iter count: 1.0 +[Apr 27, 2026 - 15:49:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:49:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 15:52:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077175 43.159401 42.987068 +peak_gpu_mem_gb 131.581049 131.620712 131.567828 +peak_gpu_mem_reserved_gb 135.527344 135.677734 135.457031 +util 71.250000 78.000000 66.000000 +[Apr 27, 2026 - 15:52:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 265 : avg iteration time 173.81 seconds +[Apr 27, 2026 - 15:52:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 265 : data loading time 0.00 +[Apr 27, 2026 - 15:52:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 265 : avg forward pass time 78.35 +[Apr 27, 2026 - 15:52:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 265 : backward pass time 8.42 +[Apr 27, 2026 - 15:52:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 265 : optimizer step time 0.34 +[Apr 27, 2026 - 15:52:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 265-------------------- +[Apr 27, 2026 - 15:52:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 206592.0762 iter count: 1.0 +[Apr 27, 2026 - 15:52:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6655 iter count: 1.0 +[Apr 27, 2026 - 15:52:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 206592.0762iter count: 1.0 +[Apr 27, 2026 - 15:52:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:52:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.7154iter count: 1.0 +[Apr 27, 2026 - 15:52:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 2.3041 iter count: 1.0 +[Apr 27, 2026 - 15:52:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.7508 iter count: 1.0 +[Apr 27, 2026 - 15:52:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 334605.4375iter count: 1.0 +[Apr 27, 2026 - 15:52:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 100.3082iter count: 1.0 +[Apr 27, 2026 - 15:52:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 5398834.5000iter count: 1.0 +[Apr 27, 2026 - 15:52:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6655iter count: 1.0 +[Apr 27, 2026 - 15:52:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.3492 iter count: 1.0 +[Apr 27, 2026 - 15:52:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 44287.2070iter count: 1.0 +[Apr 27, 2026 - 15:52:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.0893 iter count: 1.0 +[Apr 27, 2026 - 15:52:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 482.3311iter count: 1.0 +[Apr 27, 2026 - 15:52:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.7253 iter count: 1.0 +[Apr 27, 2026 - 15:52:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 12715.7646iter count: 1.0 +[Apr 27, 2026 - 15:52:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.7688 iter count: 1.0 +[Apr 27, 2026 - 15:52:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 1877.2788iter count: 1.0 +[Apr 27, 2026 - 15:52:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0158 iter count: 1.0 +[Apr 27, 2026 - 15:52:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:52:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 15:52:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077232 43.159401 42.987068 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.928711 69.996094 69.843750 +util 97.000000 99.000000 92.000000 +[Apr 27, 2026 - 15:52:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 266 : avg iteration time 52.13 seconds +[Apr 27, 2026 - 15:52:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 266 : data loading time 0.00 +[Apr 27, 2026 - 15:52:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 266 : avg forward pass time 13.48 +[Apr 27, 2026 - 15:52:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 266 : backward pass time 11.55 +[Apr 27, 2026 - 15:52:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 266 : optimizer step time 0.98 +[Apr 27, 2026 - 15:52:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 266-------------------- +[Apr 27, 2026 - 15:52:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0164 iter count: 1.0 +[Apr 27, 2026 - 15:52:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0164 iter count: 1.0 +[Apr 27, 2026 - 15:52:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:52:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 15:53:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077224 43.159401 42.987068 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.943359 70.015625 69.820312 +util 97.250000 100.000000 91.000000 +[Apr 27, 2026 - 15:53:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 267 : avg iteration time 50.68 seconds +[Apr 27, 2026 - 15:53:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 267 : data loading time 0.00 +[Apr 27, 2026 - 15:53:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 267 : avg forward pass time 12.67 +[Apr 27, 2026 - 15:53:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 267 : backward pass time 11.64 +[Apr 27, 2026 - 15:53:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 267 : optimizer step time 1.44 +[Apr 27, 2026 - 15:53:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 267-------------------- +[Apr 27, 2026 - 15:53:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0166 iter count: 1.0 +[Apr 27, 2026 - 15:53:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0166 iter count: 1.0 +[Apr 27, 2026 - 15:53:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:53:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 15:54:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077285 43.159401 42.987068 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.909180 69.996094 69.820312 +util 97.750000 100.000000 94.000000 +[Apr 27, 2026 - 15:54:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 268 : avg iteration time 52.68 seconds +[Apr 27, 2026 - 15:54:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 268 : data loading time 0.00 +[Apr 27, 2026 - 15:54:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 268 : avg forward pass time 14.23 +[Apr 27, 2026 - 15:54:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 268 : backward pass time 11.67 +[Apr 27, 2026 - 15:54:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 268 : optimizer step time 0.98 +[Apr 27, 2026 - 15:54:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 268-------------------- +[Apr 27, 2026 - 15:54:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0152 iter count: 1.0 +[Apr 27, 2026 - 15:54:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0152 iter count: 1.0 +[Apr 27, 2026 - 15:54:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:54:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 15:55:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077535 43.159401 42.987068 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.006836 70.039062 69.996094 +util 95.750000 99.000000 90.000000 +[Apr 27, 2026 - 15:55:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 269 : avg iteration time 47.16 seconds +[Apr 27, 2026 - 15:55:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 269 : data loading time 0.00 +[Apr 27, 2026 - 15:55:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 269 : avg forward pass time 11.41 +[Apr 27, 2026 - 15:55:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 269 : backward pass time 11.56 +[Apr 27, 2026 - 15:55:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 269 : optimizer step time 0.98 +[Apr 27, 2026 - 15:55:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 269-------------------- +[Apr 27, 2026 - 15:55:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0148 iter count: 1.0 +[Apr 27, 2026 - 15:55:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0148 iter count: 1.0 +[Apr 27, 2026 - 15:55:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:55:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 15:58:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077493 43.159401 42.987068 +peak_gpu_mem_gb 131.581049 131.620712 131.567828 +peak_gpu_mem_reserved_gb 135.507324 135.541016 135.476562 +util 58.250000 71.000000 50.000000 +[Apr 27, 2026 - 15:58:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 270 : avg iteration time 172.77 seconds +[Apr 27, 2026 - 15:58:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 270 : data loading time 0.00 +[Apr 27, 2026 - 15:58:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 270 : avg forward pass time 77.58 +[Apr 27, 2026 - 15:58:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 270 : backward pass time 8.91 +[Apr 27, 2026 - 15:58:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 270 : optimizer step time 0.40 +[Apr 27, 2026 - 15:58:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 270-------------------- +[Apr 27, 2026 - 15:58:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 534093.7795 iter count: 1.0 +[Apr 27, 2026 - 15:58:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7019 iter count: 1.0 +[Apr 27, 2026 - 15:58:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 534093.7795iter count: 1.0 +[Apr 27, 2026 - 15:58:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:58:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.1604iter count: 1.0 +[Apr 27, 2026 - 15:58:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.6924 iter count: 1.0 +[Apr 27, 2026 - 15:58:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.5963 iter count: 1.0 +[Apr 27, 2026 - 15:58:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 1090092.7500iter count: 1.0 +[Apr 27, 2026 - 15:58:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 3.9937 iter count: 1.0 +[Apr 27, 2026 - 15:58:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 29284300.0000iter count: 1.0 +[Apr 27, 2026 - 15:58:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7019iter count: 1.0 +[Apr 27, 2026 - 15:58:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.1976 iter count: 1.0 +[Apr 27, 2026 - 15:58:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 598.9087iter count: 1.0 +[Apr 27, 2026 - 15:58:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.6143 iter count: 1.0 +[Apr 27, 2026 - 15:58:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 75256.7266iter count: 1.0 +[Apr 27, 2026 - 15:58:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.1637 iter count: 1.0 +[Apr 27, 2026 - 15:58:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 4136.0146iter count: 1.0 +[Apr 27, 2026 - 15:58:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.6683 iter count: 1.0 +[Apr 27, 2026 - 15:58:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 11346.1270iter count: 1.0 +[Apr 27, 2026 - 15:58:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0141 iter count: 1.0 +[Apr 27, 2026 - 15:58:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:58:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 15:59:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077550 43.159401 42.987068 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.914551 69.982422 69.878906 +util 97.750000 99.000000 95.000000 +[Apr 27, 2026 - 15:59:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 271 : avg iteration time 55.38 seconds +[Apr 27, 2026 - 15:59:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 271 : data loading time 0.00 +[Apr 27, 2026 - 15:59:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 271 : avg forward pass time 14.87 +[Apr 27, 2026 - 15:59:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 271 : backward pass time 12.92 +[Apr 27, 2026 - 15:59:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 271 : optimizer step time 1.13 +[Apr 27, 2026 - 15:59:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 271-------------------- +[Apr 27, 2026 - 15:59:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0168 iter count: 1.0 +[Apr 27, 2026 - 15:59:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0168 iter count: 1.0 +[Apr 27, 2026 - 15:59:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 15:59:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 16:00:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077551 43.159424 42.987068 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.987793 70.041016 69.917969 +util 96.750000 99.000000 92.000000 +[Apr 27, 2026 - 16:00:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 272 : avg iteration time 47.34 seconds +[Apr 27, 2026 - 16:00:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 272 : data loading time 0.00 +[Apr 27, 2026 - 16:00:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 272 : avg forward pass time 11.52 +[Apr 27, 2026 - 16:00:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 272 : backward pass time 11.76 +[Apr 27, 2026 - 16:00:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 272 : optimizer step time 0.98 +[Apr 27, 2026 - 16:00:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 272-------------------- +[Apr 27, 2026 - 16:00:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0154 iter count: 1.0 +[Apr 27, 2026 - 16:00:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0154 iter count: 1.0 +[Apr 27, 2026 - 16:00:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 16:00:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 16:00:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077612 43.159420 42.987068 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.124512 70.250000 69.957031 +util 96.500000 99.000000 91.000000 +[Apr 27, 2026 - 16:00:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 273 : avg iteration time 52.80 seconds +[Apr 27, 2026 - 16:00:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 273 : data loading time 0.00 +[Apr 27, 2026 - 16:00:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 273 : avg forward pass time 14.08 +[Apr 27, 2026 - 16:00:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 273 : backward pass time 11.59 +[Apr 27, 2026 - 16:00:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 273 : optimizer step time 0.97 +[Apr 27, 2026 - 16:00:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 273-------------------- +[Apr 27, 2026 - 16:00:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0148 iter count: 1.0 +[Apr 27, 2026 - 16:00:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0148 iter count: 1.0 +[Apr 27, 2026 - 16:00:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 16:00:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 16:01:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077607 43.159401 42.987068 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.934082 70.060547 69.859375 +util 97.500000 100.000000 93.000000 +[Apr 27, 2026 - 16:01:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 274 : avg iteration time 47.77 seconds +[Apr 27, 2026 - 16:01:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 274 : data loading time 0.00 +[Apr 27, 2026 - 16:01:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 274 : avg forward pass time 11.37 +[Apr 27, 2026 - 16:01:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 274 : backward pass time 11.53 +[Apr 27, 2026 - 16:01:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 274 : optimizer step time 1.04 +[Apr 27, 2026 - 16:01:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 274-------------------- +[Apr 27, 2026 - 16:01:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0136 iter count: 1.0 +[Apr 27, 2026 - 16:01:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0136 iter count: 1.0 +[Apr 27, 2026 - 16:01:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 16:01:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 16:04:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077665 43.159424 42.987068 +peak_gpu_mem_gb 131.581049 131.620712 131.567828 +peak_gpu_mem_reserved_gb 135.577148 135.619141 135.537109 +util 70.250000 77.000000 65.000000 +[Apr 27, 2026 - 16:04:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 275 : avg iteration time 173.09 seconds +[Apr 27, 2026 - 16:04:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 275 : data loading time 0.00 +[Apr 27, 2026 - 16:04:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 275 : avg forward pass time 77.95 +[Apr 27, 2026 - 16:04:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 275 : backward pass time 8.40 +[Apr 27, 2026 - 16:04:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 275 : optimizer step time 0.33 +[Apr 27, 2026 - 16:04:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 275-------------------- +[Apr 27, 2026 - 16:04:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 6274558.5347 iter count: 1.0 +[Apr 27, 2026 - 16:04:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6861 iter count: 1.0 +[Apr 27, 2026 - 16:04:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 6274558.5347iter count: 1.0 +[Apr 27, 2026 - 16:04:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 16:04:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.7441iter count: 1.0 +[Apr 27, 2026 - 16:04:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.8392 iter count: 1.0 +[Apr 27, 2026 - 16:04:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 9.5929 iter count: 1.0 +[Apr 27, 2026 - 16:04:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 6883347.0000iter count: 1.0 +[Apr 27, 2026 - 16:04:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 5.3571 iter count: 1.0 +[Apr 27, 2026 - 16:04:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 214943632.0000iter count: 1.0 +[Apr 27, 2026 - 16:04:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6861iter count: 1.0 +[Apr 27, 2026 - 16:04:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.7155 iter count: 1.0 +[Apr 27, 2026 - 16:04:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 1687.3361iter count: 1.0 +[Apr 27, 2026 - 16:04:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.6470 iter count: 1.0 +[Apr 27, 2026 - 16:04:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 10872.4902iter count: 1.0 +[Apr 27, 2026 - 16:04:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.3516 iter count: 1.0 +[Apr 27, 2026 - 16:04:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 44494.8516iter count: 1.0 +[Apr 27, 2026 - 16:04:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.7036 iter count: 1.0 +[Apr 27, 2026 - 16:04:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 664801.5000iter count: 1.0 +[Apr 27, 2026 - 16:04:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0136 iter count: 1.0 +[Apr 27, 2026 - 16:04:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 16:04:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 16:05:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077733 43.159424 42.987114 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.958008 70.000000 69.917969 +util 96.500000 99.000000 92.000000 +[Apr 27, 2026 - 16:05:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 276 : avg iteration time 54.68 seconds +[Apr 27, 2026 - 16:05:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 276 : data loading time 0.00 +[Apr 27, 2026 - 16:05:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 276 : avg forward pass time 14.59 +[Apr 27, 2026 - 16:05:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 276 : backward pass time 11.55 +[Apr 27, 2026 - 16:05:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 276 : optimizer step time 0.99 +[Apr 27, 2026 - 16:05:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 276-------------------- +[Apr 27, 2026 - 16:05:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0171 iter count: 1.0 +[Apr 27, 2026 - 16:05:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0171 iter count: 1.0 +[Apr 27, 2026 - 16:05:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 16:05:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 16:06:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077722 43.159424 42.987068 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.972656 70.074219 69.902344 +util 96.000000 100.000000 90.000000 +[Apr 27, 2026 - 16:06:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 277 : avg iteration time 54.41 seconds +[Apr 27, 2026 - 16:06:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 277 : data loading time 0.00 +[Apr 27, 2026 - 16:06:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 277 : avg forward pass time 14.31 +[Apr 27, 2026 - 16:06:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 277 : backward pass time 12.98 +[Apr 27, 2026 - 16:06:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 277 : optimizer step time 1.23 +[Apr 27, 2026 - 16:06:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 277-------------------- +[Apr 27, 2026 - 16:06:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0171 iter count: 1.0 +[Apr 27, 2026 - 16:06:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0171 iter count: 1.0 +[Apr 27, 2026 - 16:06:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 16:06:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 16:07:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077855 43.159424 42.987099 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.870117 70.015625 69.781250 +util 97.000000 99.000000 94.000000 +[Apr 27, 2026 - 16:07:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 278 : avg iteration time 52.91 seconds +[Apr 27, 2026 - 16:07:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 278 : data loading time 0.00 +[Apr 27, 2026 - 16:07:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 278 : avg forward pass time 14.41 +[Apr 27, 2026 - 16:07:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 278 : backward pass time 11.53 +[Apr 27, 2026 - 16:07:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 278 : optimizer step time 1.02 +[Apr 27, 2026 - 16:07:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 278-------------------- +[Apr 27, 2026 - 16:07:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0174 iter count: 1.0 +[Apr 27, 2026 - 16:07:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0174 iter count: 1.0 +[Apr 27, 2026 - 16:07:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 16:07:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 16:08:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077847 43.159447 42.987068 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.948242 70.015625 69.902344 +util 97.500000 99.000000 95.000000 +[Apr 27, 2026 - 16:08:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 279 : avg iteration time 48.01 seconds +[Apr 27, 2026 - 16:08:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 279 : data loading time 0.00 +[Apr 27, 2026 - 16:08:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 279 : avg forward pass time 11.43 +[Apr 27, 2026 - 16:08:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 279 : backward pass time 11.55 +[Apr 27, 2026 - 16:08:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 279 : optimizer step time 0.99 +[Apr 27, 2026 - 16:08:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 279-------------------- +[Apr 27, 2026 - 16:08:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0146 iter count: 1.0 +[Apr 27, 2026 - 16:08:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0146 iter count: 1.0 +[Apr 27, 2026 - 16:08:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 16:08:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 16:11:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077812 43.159626 42.987293 +peak_gpu_mem_gb 131.581049 131.620712 131.567828 +peak_gpu_mem_reserved_gb 135.562012 135.638672 135.457031 +util 69.000000 76.000000 64.000000 +[Apr 27, 2026 - 16:11:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 280 : avg iteration time 178.78 seconds +[Apr 27, 2026 - 16:11:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 280 : data loading time 0.00 +[Apr 27, 2026 - 16:11:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 280 : avg forward pass time 80.86 +[Apr 27, 2026 - 16:11:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 280 : backward pass time 8.43 +[Apr 27, 2026 - 16:11:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 280 : optimizer step time 0.33 +[Apr 27, 2026 - 16:11:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 280-------------------- +[Apr 27, 2026 - 16:11:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 196279.6282 iter count: 1.0 +[Apr 27, 2026 - 16:11:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6525 iter count: 1.0 +[Apr 27, 2026 - 16:11:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 196279.6282iter count: 1.0 +[Apr 27, 2026 - 16:11:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 16:11:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.6782iter count: 1.0 +[Apr 27, 2026 - 16:11:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 2.1973 iter count: 1.0 +[Apr 27, 2026 - 16:11:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.7589 iter count: 1.0 +[Apr 27, 2026 - 16:11:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 224819.8281iter count: 1.0 +[Apr 27, 2026 - 16:11:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 81.0054iter count: 1.0 +[Apr 27, 2026 - 16:11:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 5486716.0000iter count: 1.0 +[Apr 27, 2026 - 16:11:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6525iter count: 1.0 +[Apr 27, 2026 - 16:11:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.3395 iter count: 1.0 +[Apr 27, 2026 - 16:11:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 43432.3086iter count: 1.0 +[Apr 27, 2026 - 16:11:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.0691 iter count: 1.0 +[Apr 27, 2026 - 16:11:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 25288.7012iter count: 1.0 +[Apr 27, 2026 - 16:11:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 6.4605 iter count: 1.0 +[Apr 27, 2026 - 16:11:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 408818.3438iter count: 1.0 +[Apr 27, 2026 - 16:11:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.7979 iter count: 1.0 +[Apr 27, 2026 - 16:11:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 1989.6359iter count: 1.0 +[Apr 27, 2026 - 16:11:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0139 iter count: 1.0 +[Apr 27, 2026 - 16:11:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 16:11:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 16:11:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.077803 43.159626 42.987293 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.797363 69.878906 69.552734 +util 96.250000 99.000000 91.000000 +[Apr 27, 2026 - 16:11:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 281 : avg iteration time 49.29 seconds +[Apr 27, 2026 - 16:11:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 281 : data loading time 0.00 +[Apr 27, 2026 - 16:11:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 281 : avg forward pass time 11.90 +[Apr 27, 2026 - 16:11:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 281 : backward pass time 11.61 +[Apr 27, 2026 - 16:11:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 281 : optimizer step time 0.98 +[Apr 27, 2026 - 16:11:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 281-------------------- +[Apr 27, 2026 - 16:11:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0163 iter count: 1.0 +[Apr 27, 2026 - 16:11:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0163 iter count: 1.0 +[Apr 27, 2026 - 16:11:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 16:11:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 16:12:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.078264 43.159649 42.987316 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.875488 69.996094 69.552734 +util 96.250000 100.000000 90.000000 +[Apr 27, 2026 - 16:12:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 282 : avg iteration time 54.05 seconds +[Apr 27, 2026 - 16:12:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 282 : data loading time 0.00 +[Apr 27, 2026 - 16:12:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 282 : avg forward pass time 14.21 +[Apr 27, 2026 - 16:12:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 282 : backward pass time 12.84 +[Apr 27, 2026 - 16:12:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 282 : optimizer step time 1.24 +[Apr 27, 2026 - 16:12:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 282-------------------- +[Apr 27, 2026 - 16:12:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0174 iter count: 1.0 +[Apr 27, 2026 - 16:12:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0174 iter count: 1.0 +[Apr 27, 2026 - 16:12:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 16:12:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 16:13:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.078253 43.159626 42.987293 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.987793 70.113281 69.923828 +util 97.000000 99.000000 94.000000 +[Apr 27, 2026 - 16:13:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 283 : avg iteration time 52.67 seconds +[Apr 27, 2026 - 16:13:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 283 : data loading time 0.00 +[Apr 27, 2026 - 16:13:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 283 : avg forward pass time 14.26 +[Apr 27, 2026 - 16:13:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 283 : backward pass time 11.53 +[Apr 27, 2026 - 16:13:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 283 : optimizer step time 1.01 +[Apr 27, 2026 - 16:13:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 283-------------------- +[Apr 27, 2026 - 16:13:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0190 iter count: 1.0 +[Apr 27, 2026 - 16:13:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0190 iter count: 1.0 +[Apr 27, 2026 - 16:13:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 16:13:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 16:14:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.078158 43.159626 42.987293 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.958496 70.035156 69.859375 +util 96.250000 99.000000 91.000000 +[Apr 27, 2026 - 16:14:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 284 : avg iteration time 50.43 seconds +[Apr 27, 2026 - 16:14:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 284 : data loading time 0.00 +[Apr 27, 2026 - 16:14:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 284 : avg forward pass time 12.51 +[Apr 27, 2026 - 16:14:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 284 : backward pass time 11.57 +[Apr 27, 2026 - 16:14:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 284 : optimizer step time 0.99 +[Apr 27, 2026 - 16:14:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 284-------------------- +[Apr 27, 2026 - 16:14:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0171 iter count: 1.0 +[Apr 27, 2026 - 16:14:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0171 iter count: 1.0 +[Apr 27, 2026 - 16:14:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 16:14:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 16:17:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.078122 43.159626 42.987293 +peak_gpu_mem_gb 131.581049 131.620712 131.567828 +peak_gpu_mem_reserved_gb 135.552734 135.580078 135.537109 +util 65.500000 71.000000 61.000000 +[Apr 27, 2026 - 16:17:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 285 : avg iteration time 173.46 seconds +[Apr 27, 2026 - 16:17:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 285 : data loading time 0.00 +[Apr 27, 2026 - 16:17:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 285 : avg forward pass time 78.15 +[Apr 27, 2026 - 16:17:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 285 : backward pass time 8.49 +[Apr 27, 2026 - 16:17:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 285 : optimizer step time 0.35 +[Apr 27, 2026 - 16:17:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 285-------------------- +[Apr 27, 2026 - 16:17:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 95036.5464 iter count: 1.0 +[Apr 27, 2026 - 16:17:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6827 iter count: 1.0 +[Apr 27, 2026 - 16:17:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 95036.5464iter count: 1.0 +[Apr 27, 2026 - 16:17:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 16:17:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.6394iter count: 1.0 +[Apr 27, 2026 - 16:17:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 2.2181 iter count: 1.0 +[Apr 27, 2026 - 16:17:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.0226 iter count: 1.0 +[Apr 27, 2026 - 16:17:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 143975.9062iter count: 1.0 +[Apr 27, 2026 - 16:17:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 84.4541iter count: 1.0 +[Apr 27, 2026 - 16:17:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 1258188.0000iter count: 1.0 +[Apr 27, 2026 - 16:17:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6827iter count: 1.0 +[Apr 27, 2026 - 16:17:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.7252 iter count: 1.0 +[Apr 27, 2026 - 16:17:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 93932.2344iter count: 1.0 +[Apr 27, 2026 - 16:17:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 6.3249 iter count: 1.0 +[Apr 27, 2026 - 16:17:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 311688.4062iter count: 1.0 +[Apr 27, 2026 - 16:17:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 2.9714 iter count: 1.0 +[Apr 27, 2026 - 16:17:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 381.0295iter count: 1.0 +[Apr 27, 2026 - 16:17:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.6201 iter count: 1.0 +[Apr 27, 2026 - 16:17:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 10303.2168iter count: 1.0 +[Apr 27, 2026 - 16:17:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0218 iter count: 1.0 +[Apr 27, 2026 - 16:17:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 16:17:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 16:18:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.078139 43.159649 42.987316 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.977539 70.035156 69.859375 +util 97.750000 99.000000 94.000000 +[Apr 27, 2026 - 16:18:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 286 : avg iteration time 48.22 seconds +[Apr 27, 2026 - 16:18:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 286 : data loading time 0.00 +[Apr 27, 2026 - 16:18:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 286 : avg forward pass time 12.00 +[Apr 27, 2026 - 16:18:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 286 : backward pass time 11.57 +[Apr 27, 2026 - 16:18:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 286 : optimizer step time 0.98 +[Apr 27, 2026 - 16:18:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 286-------------------- +[Apr 27, 2026 - 16:18:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0201 iter count: 1.0 +[Apr 27, 2026 - 16:18:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0201 iter count: 1.0 +[Apr 27, 2026 - 16:18:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 16:18:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 16:19:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.078370 43.159626 42.987293 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.992188 70.035156 69.937500 +util 96.750000 100.000000 92.000000 +[Apr 27, 2026 - 16:19:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 287 : avg iteration time 53.15 seconds +[Apr 27, 2026 - 16:19:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 287 : data loading time 0.00 +[Apr 27, 2026 - 16:19:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 287 : avg forward pass time 13.88 +[Apr 27, 2026 - 16:19:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 287 : backward pass time 11.64 +[Apr 27, 2026 - 16:19:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 287 : optimizer step time 1.42 +[Apr 27, 2026 - 16:19:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 287-------------------- +[Apr 27, 2026 - 16:19:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0199 iter count: 1.0 +[Apr 27, 2026 - 16:19:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0199 iter count: 1.0 +[Apr 27, 2026 - 16:19:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 16:19:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 16:20:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.078497 43.159626 42.987297 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.997070 70.035156 69.921875 +util 98.000000 100.000000 94.000000 +[Apr 27, 2026 - 16:20:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 288 : avg iteration time 52.79 seconds +[Apr 27, 2026 - 16:20:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 288 : data loading time 0.00 +[Apr 27, 2026 - 16:20:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 288 : avg forward pass time 14.30 +[Apr 27, 2026 - 16:20:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 288 : backward pass time 11.67 +[Apr 27, 2026 - 16:20:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 288 : optimizer step time 0.98 +[Apr 27, 2026 - 16:20:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 288-------------------- +[Apr 27, 2026 - 16:20:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0208 iter count: 1.0 +[Apr 27, 2026 - 16:20:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0208 iter count: 1.0 +[Apr 27, 2026 - 16:20:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 16:20:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 16:20:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.078449 43.159626 42.987293 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.036133 70.074219 70.000000 +util 96.000000 100.000000 91.000000 +[Apr 27, 2026 - 16:20:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 289 : avg iteration time 53.31 seconds +[Apr 27, 2026 - 16:20:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 289 : data loading time 0.00 +[Apr 27, 2026 - 16:20:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 289 : avg forward pass time 14.18 +[Apr 27, 2026 - 16:20:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 289 : backward pass time 11.56 +[Apr 27, 2026 - 16:20:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 289 : optimizer step time 0.98 +[Apr 27, 2026 - 16:20:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 289-------------------- +[Apr 27, 2026 - 16:20:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0186 iter count: 1.0 +[Apr 27, 2026 - 16:20:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0186 iter count: 1.0 +[Apr 27, 2026 - 16:20:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 16:20:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 16:23:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.078125 43.159664 42.987293 +peak_gpu_mem_gb 131.581049 131.620712 131.567828 +peak_gpu_mem_reserved_gb 135.557617 135.576172 135.537109 +util 72.250000 79.000000 67.000000 +[Apr 27, 2026 - 16:23:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 290 : avg iteration time 176.86 seconds +[Apr 27, 2026 - 16:23:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 290 : data loading time 0.00 +[Apr 27, 2026 - 16:23:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 290 : avg forward pass time 79.58 +[Apr 27, 2026 - 16:23:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 290 : backward pass time 8.41 +[Apr 27, 2026 - 16:23:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 290 : optimizer step time 0.34 +[Apr 27, 2026 - 16:23:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 290-------------------- +[Apr 27, 2026 - 16:23:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 203001.5713 iter count: 1.0 +[Apr 27, 2026 - 16:23:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6185 iter count: 1.0 +[Apr 27, 2026 - 16:23:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 203001.5713iter count: 1.0 +[Apr 27, 2026 - 16:23:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 16:23:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.8216iter count: 1.0 +[Apr 27, 2026 - 16:23:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.7713 iter count: 1.0 +[Apr 27, 2026 - 16:23:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.1117 iter count: 1.0 +[Apr 27, 2026 - 16:23:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 440232.9375iter count: 1.0 +[Apr 27, 2026 - 16:23:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 34.5548iter count: 1.0 +[Apr 27, 2026 - 16:23:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 11110064.0000iter count: 1.0 +[Apr 27, 2026 - 16:23:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6185iter count: 1.0 +[Apr 27, 2026 - 16:23:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.3754 iter count: 1.0 +[Apr 27, 2026 - 16:23:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 854.7172iter count: 1.0 +[Apr 27, 2026 - 16:23:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 1.7713 iter count: 1.0 +[Apr 27, 2026 - 16:23:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 34.5548 iter count: 1.0 +[Apr 27, 2026 - 16:23:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.5883 iter count: 1.0 +[Apr 27, 2026 - 16:23:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 1308.5586iter count: 1.0 +[Apr 27, 2026 - 16:23:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.5006 iter count: 1.0 +[Apr 27, 2026 - 16:23:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 8113.1982iter count: 1.0 +[Apr 27, 2026 - 16:23:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0204 iter count: 1.0 +[Apr 27, 2026 - 16:23:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 16:23:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 16:24:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.078188 43.159664 42.987293 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.953613 70.001953 69.917969 +util 97.750000 99.000000 94.000000 +[Apr 27, 2026 - 16:24:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 291 : avg iteration time 48.03 seconds +[Apr 27, 2026 - 16:24:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 291 : data loading time 0.00 +[Apr 27, 2026 - 16:24:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 291 : avg forward pass time 11.94 +[Apr 27, 2026 - 16:24:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 291 : backward pass time 11.54 +[Apr 27, 2026 - 16:24:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 291 : optimizer step time 0.98 +[Apr 27, 2026 - 16:24:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 291-------------------- +[Apr 27, 2026 - 16:24:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0179 iter count: 1.0 +[Apr 27, 2026 - 16:24:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0179 iter count: 1.0 +[Apr 27, 2026 - 16:24:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 16:24:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 16:25:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.078182 43.159641 42.987293 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.963379 70.021484 69.878906 +util 98.750000 100.000000 96.000000 +[Apr 27, 2026 - 16:25:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 292 : avg iteration time 53.14 seconds +[Apr 27, 2026 - 16:25:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 292 : data loading time 0.00 +[Apr 27, 2026 - 16:25:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 292 : avg forward pass time 13.93 +[Apr 27, 2026 - 16:25:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 292 : backward pass time 11.55 +[Apr 27, 2026 - 16:25:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 292 : optimizer step time 1.26 +[Apr 27, 2026 - 16:25:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 292-------------------- +[Apr 27, 2026 - 16:25:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0186 iter count: 1.0 +[Apr 27, 2026 - 16:25:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0186 iter count: 1.0 +[Apr 27, 2026 - 16:25:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 16:25:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 16:26:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.078184 43.159649 42.987293 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.963379 70.021484 69.839844 +util 96.750000 100.000000 92.000000 +[Apr 27, 2026 - 16:26:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 293 : avg iteration time 50.09 seconds +[Apr 27, 2026 - 16:26:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 293 : data loading time 0.00 +[Apr 27, 2026 - 16:26:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 293 : avg forward pass time 12.59 +[Apr 27, 2026 - 16:26:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 293 : backward pass time 12.32 +[Apr 27, 2026 - 16:26:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 293 : optimizer step time 1.00 +[Apr 27, 2026 - 16:26:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 293-------------------- +[Apr 27, 2026 - 16:26:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0146 iter count: 1.0 +[Apr 27, 2026 - 16:26:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0146 iter count: 1.0 +[Apr 27, 2026 - 16:26:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 16:26:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 16:27:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.078251 43.159664 42.987293 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.875488 70.021484 69.800781 +util 97.500000 100.000000 94.000000 +[Apr 27, 2026 - 16:27:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 294 : avg iteration time 48.91 seconds +[Apr 27, 2026 - 16:27:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 294 : data loading time 0.00 +[Apr 27, 2026 - 16:27:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 294 : avg forward pass time 12.42 +[Apr 27, 2026 - 16:27:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 294 : backward pass time 11.57 +[Apr 27, 2026 - 16:27:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 294 : optimizer step time 0.97 +[Apr 27, 2026 - 16:27:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 294-------------------- +[Apr 27, 2026 - 16:27:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0159 iter count: 1.0 +[Apr 27, 2026 - 16:27:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0159 iter count: 1.0 +[Apr 27, 2026 - 16:27:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 16:27:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 16:30:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.078138 43.159641 42.987312 +peak_gpu_mem_gb 131.581049 131.620712 131.567828 +peak_gpu_mem_reserved_gb 135.533203 135.619141 135.478516 +util 71.250000 78.000000 64.000000 +[Apr 27, 2026 - 16:30:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 295 : avg iteration time 175.76 seconds +[Apr 27, 2026 - 16:30:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 295 : data loading time 0.00 +[Apr 27, 2026 - 16:30:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 295 : avg forward pass time 79.19 +[Apr 27, 2026 - 16:30:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 295 : backward pass time 8.44 +[Apr 27, 2026 - 16:30:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 295 : optimizer step time 0.34 +[Apr 27, 2026 - 16:30:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 295-------------------- +[Apr 27, 2026 - 16:30:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 162482.5601 iter count: 1.0 +[Apr 27, 2026 - 16:30:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5908 iter count: 1.0 +[Apr 27, 2026 - 16:30:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 162482.5601iter count: 1.0 +[Apr 27, 2026 - 16:30:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 16:30:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.3838iter count: 1.0 +[Apr 27, 2026 - 16:30:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.8216 iter count: 1.0 +[Apr 27, 2026 - 16:30:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.7311 iter count: 1.0 +[Apr 27, 2026 - 16:30:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 283189.7812iter count: 1.0 +[Apr 27, 2026 - 16:30:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 5.1714 iter count: 1.0 +[Apr 27, 2026 - 16:30:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 5189454.5000iter count: 1.0 +[Apr 27, 2026 - 16:30:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5908iter count: 1.0 +[Apr 27, 2026 - 16:30:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.7448 iter count: 1.0 +[Apr 27, 2026 - 16:30:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 13221.7471iter count: 1.0 +[Apr 27, 2026 - 16:30:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.2949 iter count: 1.0 +[Apr 27, 2026 - 16:30:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 5376.7939iter count: 1.0 +[Apr 27, 2026 - 16:30:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 2.6842 iter count: 1.0 +[Apr 27, 2026 - 16:30:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 214.5093iter count: 1.0 +[Apr 27, 2026 - 16:30:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.6619 iter count: 1.0 +[Apr 27, 2026 - 16:30:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 11200.9229iter count: 1.0 +[Apr 27, 2026 - 16:30:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0159 iter count: 1.0 +[Apr 27, 2026 - 16:30:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 16:30:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 16:30:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.078134 43.159641 42.987293 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.958496 69.996094 69.898438 +util 96.500000 99.000000 92.000000 +[Apr 27, 2026 - 16:30:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 296 : avg iteration time 48.07 seconds +[Apr 27, 2026 - 16:30:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 296 : data loading time 0.00 +[Apr 27, 2026 - 16:30:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 296 : avg forward pass time 11.96 +[Apr 27, 2026 - 16:30:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 296 : backward pass time 11.55 +[Apr 27, 2026 - 16:30:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 296 : optimizer step time 0.99 +[Apr 27, 2026 - 16:30:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 296-------------------- +[Apr 27, 2026 - 16:30:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0155 iter count: 1.0 +[Apr 27, 2026 - 16:30:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0155 iter count: 1.0 +[Apr 27, 2026 - 16:30:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 16:30:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 16:31:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.078134 43.159641 42.987293 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.973145 70.035156 69.943359 +util 97.750000 100.000000 93.000000 +[Apr 27, 2026 - 16:31:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 297 : avg iteration time 47.53 seconds +[Apr 27, 2026 - 16:31:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 297 : data loading time 0.00 +[Apr 27, 2026 - 16:31:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 297 : avg forward pass time 11.09 +[Apr 27, 2026 - 16:31:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 297 : backward pass time 11.54 +[Apr 27, 2026 - 16:31:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 297 : optimizer step time 0.98 +[Apr 27, 2026 - 16:31:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 297-------------------- +[Apr 27, 2026 - 16:31:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0165 iter count: 1.0 +[Apr 27, 2026 - 16:31:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0165 iter count: 1.0 +[Apr 27, 2026 - 16:31:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 16:31:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 16:32:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.078139 43.159664 42.987293 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.992676 70.015625 69.957031 +util 96.000000 100.000000 91.000000 +[Apr 27, 2026 - 16:32:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 298 : avg iteration time 54.10 seconds +[Apr 27, 2026 - 16:32:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 298 : data loading time 0.00 +[Apr 27, 2026 - 16:32:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 298 : avg forward pass time 14.24 +[Apr 27, 2026 - 16:32:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 298 : backward pass time 12.86 +[Apr 27, 2026 - 16:32:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 298 : optimizer step time 1.23 +[Apr 27, 2026 - 16:32:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 298-------------------- +[Apr 27, 2026 - 16:32:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0178 iter count: 1.0 +[Apr 27, 2026 - 16:32:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0178 iter count: 1.0 +[Apr 27, 2026 - 16:32:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 16:32:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 16:33:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.078139 43.159641 42.987312 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.934082 69.957031 69.917969 +util 97.750000 99.000000 96.000000 +[Apr 27, 2026 - 16:33:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 299 : avg iteration time 49.73 seconds +[Apr 27, 2026 - 16:33:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 299 : data loading time 0.02 +[Apr 27, 2026 - 16:33:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 299 : avg forward pass time 12.81 +[Apr 27, 2026 - 16:33:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 299 : backward pass time 11.56 +[Apr 27, 2026 - 16:33:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 299 : optimizer step time 1.00 +[Apr 27, 2026 - 16:33:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 299-------------------- +[Apr 27, 2026 - 16:33:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0161 iter count: 1.0 +[Apr 27, 2026 - 16:33:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0161 iter count: 1.0 +[Apr 27, 2026 - 16:33:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 16:33:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 16:36:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.078385 43.159641 42.987293 +peak_gpu_mem_gb 131.581049 131.620712 131.567828 +peak_gpu_mem_reserved_gb 135.562012 135.580078 135.554688 +util 68.250000 79.000000 56.000000 +[Apr 27, 2026 - 16:36:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 300 : avg iteration time 175.92 seconds +[Apr 27, 2026 - 16:36:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 300 : data loading time 0.00 +[Apr 27, 2026 - 16:36:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 300 : avg forward pass time 79.20 +[Apr 27, 2026 - 16:36:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 300 : backward pass time 8.83 +[Apr 27, 2026 - 16:36:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 300 : optimizer step time 0.34 +[Apr 27, 2026 - 16:36:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 300-------------------- +[Apr 27, 2026 - 16:36:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 738791.4531 iter count: 1.0 +[Apr 27, 2026 - 16:36:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6677 iter count: 1.0 +[Apr 27, 2026 - 16:36:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 738791.4531iter count: 1.0 +[Apr 27, 2026 - 16:36:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 16:36:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.7919iter count: 1.0 +[Apr 27, 2026 - 16:36:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.6876 iter count: 1.0 +[Apr 27, 2026 - 16:36:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.5259 iter count: 1.0 +[Apr 27, 2026 - 16:36:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 1349333.1250iter count: 1.0 +[Apr 27, 2026 - 16:36:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 3.9561 iter count: 1.0 +[Apr 27, 2026 - 16:36:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 25438662.0000iter count: 1.0 +[Apr 27, 2026 - 16:36:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6677iter count: 1.0 +[Apr 27, 2026 - 16:36:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 7.7817 iter count: 1.0 +[Apr 27, 2026 - 16:36:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 5742761.5000iter count: 1.0 +[Apr 27, 2026 - 16:36:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 6.9457 iter count: 1.0 +[Apr 27, 2026 - 16:36:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 1078903.3750iter count: 1.0 +[Apr 27, 2026 - 16:36:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 6.7397 iter count: 1.0 +[Apr 27, 2026 - 16:36:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 714573.1875iter count: 1.0 +[Apr 27, 2026 - 16:36:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 7.6381 iter count: 1.0 +[Apr 27, 2026 - 16:36:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 4309181.0000iter count: 1.0 +[Apr 27, 2026 - 16:36:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0164 iter count: 1.0 +[Apr 27, 2026 - 16:36:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[Apr 27, 2026 - 16:36:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 14.59 seconds +[Apr 27, 2026 - 16:36:37 | INFO | fastgen.trainer:save_checkpoint:266 ] Saving checkpoint iteration 300 +[Apr 27, 2026 - 16:36:37 | INFO | fastgen.utils.checkpointer:save:301 ] Saving FSDP model to prefix /tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000300 +[Apr 27, 2026 - 16:36:53 | SUCCESS | fastgen.utils.checkpointer:save:329 ] Model saved at iteration 300 +[Apr 27, 2026 - 16:36:54 | INFO | fastgen.trainer:validate:358 ] Validation iteration 300 +[Apr 27, 2026 - 16:36:54 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 16:36:56 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 2.1s +[Apr 27, 2026 - 16:36:59 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 0: sync_c=6.299 +[Apr 27, 2026 - 16:37:00 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 16:37:02 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.8s +[Apr 27, 2026 - 16:37:05 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 1: sync_c=5.921 +[Apr 27, 2026 - 16:37:05 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 16:37:07 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.8s +[Apr 27, 2026 - 16:37:10 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 2: sync_c=5.370 +[Apr 27, 2026 - 16:37:10 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 16:37:12 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.8s +[Apr 27, 2026 - 16:37:15 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 3: sync_c=4.817 +[Apr 27, 2026 - 16:37:15 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 16:37:17 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.8s +[Apr 27, 2026 - 16:37:20 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 4: sync_c=5.461 +[Apr 27, 2026 - 16:37:20 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 16:37:22 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.8s +[Apr 27, 2026 - 16:37:25 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 5: sync_c=6.755 +[Apr 27, 2026 - 16:37:25 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 16:37:27 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.8s +[Apr 27, 2026 - 16:37:30 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 6: sync_c=6.820 +[Apr 27, 2026 - 16:37:31 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 16:37:32 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.8s +[Apr 27, 2026 - 16:37:35 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 7: sync_c=4.870 +[Apr 27, 2026 - 16:37:36 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 16:37:37 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.8s +[Apr 27, 2026 - 16:37:40 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 8: sync_c=6.363 +[Apr 27, 2026 - 16:37:41 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 16:37:43 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.8s +[Apr 27, 2026 - 16:37:46 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 9: sync_c=3.695 +[Apr 27, 2026 - 16:37:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging val0 stats at iteration 300-------------------- +[Apr 27, 2026 - 16:37:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0000 iter count: 10.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[Apr 27, 2026 - 16:38:03 | INFO | fastgen.callbacks.wandb:on_validation_end:763 ] Logged 10 val videos at iteration 300 +[Apr 27, 2026 - 16:38:03 | INFO | fastgen.callbacks.wandb:on_validation_end:769 ] [SyncEval] val0 mean sync_c: 5.637 (n=10) +[Apr 27, 2026 - 16:38:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.079652 43.160545 42.987221 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.153320 70.390625 70.074219 +util 98.250000 100.000000 95.000000 +[Apr 27, 2026 - 16:38:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 301 : avg iteration time 150.21 seconds +[Apr 27, 2026 - 16:38:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 301 : data loading time 0.00 +[Apr 27, 2026 - 16:38:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 301 : avg forward pass time 12.30 +[Apr 27, 2026 - 16:38:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 301 : backward pass time 11.59 +[Apr 27, 2026 - 16:38:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 301 : optimizer step time 0.98 +[Apr 27, 2026 - 16:38:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 301-------------------- +[Apr 27, 2026 - 16:38:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0135 iter count: 1.0 +[Apr 27, 2026 - 16:38:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0135 iter count: 1.0 +[Apr 27, 2026 - 16:38:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 16:38:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 16:39:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.079653 43.160545 42.987221 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.148438 70.566406 69.996094 +util 96.500000 99.000000 92.000000 +[Apr 27, 2026 - 16:39:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 302 : avg iteration time 50.36 seconds +[Apr 27, 2026 - 16:39:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 302 : data loading time 0.00 +[Apr 27, 2026 - 16:39:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 302 : avg forward pass time 12.56 +[Apr 27, 2026 - 16:39:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 302 : backward pass time 11.59 +[Apr 27, 2026 - 16:39:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 302 : optimizer step time 1.11 +[Apr 27, 2026 - 16:39:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 302-------------------- +[Apr 27, 2026 - 16:39:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0165 iter count: 1.0 +[Apr 27, 2026 - 16:39:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0165 iter count: 1.0 +[Apr 27, 2026 - 16:39:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 16:39:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 16:40:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.079413 43.160549 42.987221 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.124023 70.468750 69.996094 +util 96.500000 99.000000 92.000000 +[Apr 27, 2026 - 16:40:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 303 : avg iteration time 50.41 seconds +[Apr 27, 2026 - 16:40:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 303 : data loading time 0.00 +[Apr 27, 2026 - 16:40:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 303 : avg forward pass time 12.76 +[Apr 27, 2026 - 16:40:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 303 : backward pass time 12.32 +[Apr 27, 2026 - 16:40:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 303 : optimizer step time 0.97 +[Apr 27, 2026 - 16:40:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 303-------------------- +[Apr 27, 2026 - 16:40:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0140 iter count: 1.0 +[Apr 27, 2026 - 16:40:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0140 iter count: 1.0 +[Apr 27, 2026 - 16:40:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 16:40:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 16:41:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.079420 43.160549 42.987225 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.172852 70.546875 69.996094 +util 96.750000 99.000000 93.000000 +[Apr 27, 2026 - 16:41:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 304 : avg iteration time 52.11 seconds +[Apr 27, 2026 - 16:41:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 304 : data loading time 0.00 +[Apr 27, 2026 - 16:41:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 304 : avg forward pass time 13.97 +[Apr 27, 2026 - 16:41:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 304 : backward pass time 11.59 +[Apr 27, 2026 - 16:41:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 304 : optimizer step time 0.97 +[Apr 27, 2026 - 16:41:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 304-------------------- +[Apr 27, 2026 - 16:41:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0170 iter count: 1.0 +[Apr 27, 2026 - 16:41:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0170 iter count: 1.0 +[Apr 27, 2026 - 16:41:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 16:41:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 16:44:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.079209 43.160999 42.987675 +peak_gpu_mem_gb 131.581049 131.620712 131.567828 +peak_gpu_mem_reserved_gb 135.552734 135.576172 135.517578 +util 75.000000 79.000000 70.000000 +[Apr 27, 2026 - 16:44:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 305 : avg iteration time 180.62 seconds +[Apr 27, 2026 - 16:44:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 305 : data loading time 0.00 +[Apr 27, 2026 - 16:44:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 305 : avg forward pass time 81.28 +[Apr 27, 2026 - 16:44:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 305 : backward pass time 9.42 +[Apr 27, 2026 - 16:44:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 305 : optimizer step time 0.34 +[Apr 27, 2026 - 16:44:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 305-------------------- +[Apr 27, 2026 - 16:44:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 155142.8418 iter count: 1.0 +[Apr 27, 2026 - 16:44:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6572 iter count: 1.0 +[Apr 27, 2026 - 16:44:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 155142.8418iter count: 1.0 +[Apr 27, 2026 - 16:44:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 16:44:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.7101iter count: 1.0 +[Apr 27, 2026 - 16:44:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.5802 iter count: 1.0 +[Apr 27, 2026 - 16:44:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.4077 iter count: 1.0 +[Apr 27, 2026 - 16:44:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 235754.4062iter count: 1.0 +[Apr 27, 2026 - 16:44:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 23.5790iter count: 1.0 +[Apr 27, 2026 - 16:44:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 2717823.5000iter count: 1.0 +[Apr 27, 2026 - 16:44:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6572iter count: 1.0 +[Apr 27, 2026 - 16:44:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.6845 iter count: 1.0 +[Apr 27, 2026 - 16:44:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 86590.3984iter count: 1.0 +[Apr 27, 2026 - 16:44:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.5822 iter count: 1.0 +[Apr 27, 2026 - 16:44:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 70571.3203iter count: 1.0 +[Apr 27, 2026 - 16:44:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.8415 iter count: 1.0 +[Apr 27, 2026 - 16:44:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 118550.6875iter count: 1.0 +[Apr 27, 2026 - 16:44:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 2.9614 iter count: 1.0 +[Apr 27, 2026 - 16:44:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 373.4460iter count: 1.0 +[Apr 27, 2026 - 16:44:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0158 iter count: 1.0 +[Apr 27, 2026 - 16:44:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 16:44:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 16:45:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.079214 43.161018 42.987675 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.041504 70.152344 69.923828 +util 97.000000 99.000000 92.000000 +[Apr 27, 2026 - 16:45:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 306 : avg iteration time 49.14 seconds +[Apr 27, 2026 - 16:45:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 306 : data loading time 0.00 +[Apr 27, 2026 - 16:45:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 306 : avg forward pass time 12.15 +[Apr 27, 2026 - 16:45:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 306 : backward pass time 12.28 +[Apr 27, 2026 - 16:45:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 306 : optimizer step time 0.99 +[Apr 27, 2026 - 16:45:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 306-------------------- +[Apr 27, 2026 - 16:45:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0141 iter count: 1.0 +[Apr 27, 2026 - 16:45:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0141 iter count: 1.0 +[Apr 27, 2026 - 16:45:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 16:45:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 16:46:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.079519 43.160999 42.987675 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.021973 70.080078 69.976562 +util 96.750000 99.000000 92.000000 +[Apr 27, 2026 - 16:46:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 307 : avg iteration time 48.99 seconds +[Apr 27, 2026 - 16:46:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 307 : data loading time 0.00 +[Apr 27, 2026 - 16:46:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 307 : avg forward pass time 12.44 +[Apr 27, 2026 - 16:46:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 307 : backward pass time 11.58 +[Apr 27, 2026 - 16:46:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 307 : optimizer step time 0.97 +[Apr 27, 2026 - 16:46:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 307-------------------- +[Apr 27, 2026 - 16:46:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0195 iter count: 1.0 +[Apr 27, 2026 - 16:46:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0195 iter count: 1.0 +[Apr 27, 2026 - 16:46:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 16:46:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 16:46:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.079578 43.160999 42.987675 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.007324 70.054688 69.943359 +util 97.500000 100.000000 92.000000 +[Apr 27, 2026 - 16:46:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 308 : avg iteration time 50.80 seconds +[Apr 27, 2026 - 16:46:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 308 : data loading time 0.00 +[Apr 27, 2026 - 16:46:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 308 : avg forward pass time 12.80 +[Apr 27, 2026 - 16:46:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 308 : backward pass time 12.08 +[Apr 27, 2026 - 16:46:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 308 : optimizer step time 1.56 +[Apr 27, 2026 - 16:46:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 308-------------------- +[Apr 27, 2026 - 16:46:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0139 iter count: 1.0 +[Apr 27, 2026 - 16:46:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0139 iter count: 1.0 +[Apr 27, 2026 - 16:46:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 16:46:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 16:47:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.079585 43.161007 42.987694 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.992676 70.015625 69.962891 +util 95.750000 99.000000 91.000000 +[Apr 27, 2026 - 16:47:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 309 : avg iteration time 49.88 seconds +[Apr 27, 2026 - 16:47:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 309 : data loading time 0.00 +[Apr 27, 2026 - 16:47:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 309 : avg forward pass time 12.65 +[Apr 27, 2026 - 16:47:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 309 : backward pass time 12.03 +[Apr 27, 2026 - 16:47:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 309 : optimizer step time 0.99 +[Apr 27, 2026 - 16:47:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 309-------------------- +[Apr 27, 2026 - 16:47:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0142 iter count: 1.0 +[Apr 27, 2026 - 16:47:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0142 iter count: 1.0 +[Apr 27, 2026 - 16:47:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 16:47:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 16:50:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.079516 43.161022 42.987675 +peak_gpu_mem_gb 131.581049 131.620712 131.567828 +peak_gpu_mem_reserved_gb 135.566406 135.638672 135.478516 +util 71.000000 76.000000 65.000000 +[Apr 27, 2026 - 16:50:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 310 : avg iteration time 170.89 seconds +[Apr 27, 2026 - 16:50:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 310 : data loading time 0.00 +[Apr 27, 2026 - 16:50:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 310 : avg forward pass time 76.90 +[Apr 27, 2026 - 16:50:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 310 : backward pass time 8.41 +[Apr 27, 2026 - 16:50:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 310 : optimizer step time 0.33 +[Apr 27, 2026 - 16:50:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 310-------------------- +[Apr 27, 2026 - 16:50:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 438264.7188 iter count: 1.0 +[Apr 27, 2026 - 16:50:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6346 iter count: 1.0 +[Apr 27, 2026 - 16:50:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 438264.7188iter count: 1.0 +[Apr 27, 2026 - 16:50:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 16:50:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 5.1616iter count: 1.0 +[Apr 27, 2026 - 16:50:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 2.9889 iter count: 1.0 +[Apr 27, 2026 - 16:50:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.0409 iter count: 1.0 +[Apr 27, 2026 - 16:50:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 566327.1250iter count: 1.0 +[Apr 27, 2026 - 16:50:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 394.6091iter count: 1.0 +[Apr 27, 2026 - 16:50:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 9644310.0000iter count: 1.0 +[Apr 27, 2026 - 16:50:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6346iter count: 1.0 +[Apr 27, 2026 - 16:50:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.8454 iter count: 1.0 +[Apr 27, 2026 - 16:50:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 16166.9043iter count: 1.0 +[Apr 27, 2026 - 16:50:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.7686 iter count: 1.0 +[Apr 27, 2026 - 16:50:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 102453.5391iter count: 1.0 +[Apr 27, 2026 - 16:50:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 6.9202 iter count: 1.0 +[Apr 27, 2026 - 16:50:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 1025276.8125iter count: 1.0 +[Apr 27, 2026 - 16:50:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.1480 iter count: 1.0 +[Apr 27, 2026 - 16:50:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 29611.7090iter count: 1.0 +[Apr 27, 2026 - 16:50:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0162 iter count: 1.0 +[Apr 27, 2026 - 16:50:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 16:50:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 16:51:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.079506 43.160999 42.987675 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.963379 69.996094 69.884766 +util 96.500000 100.000000 91.000000 +[Apr 27, 2026 - 16:51:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 311 : avg iteration time 52.17 seconds +[Apr 27, 2026 - 16:51:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 311 : data loading time 0.00 +[Apr 27, 2026 - 16:51:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 311 : avg forward pass time 13.49 +[Apr 27, 2026 - 16:51:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 311 : backward pass time 11.98 +[Apr 27, 2026 - 16:51:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 311 : optimizer step time 1.55 +[Apr 27, 2026 - 16:51:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 311-------------------- +[Apr 27, 2026 - 16:51:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0161 iter count: 1.0 +[Apr 27, 2026 - 16:51:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0161 iter count: 1.0 +[Apr 27, 2026 - 16:51:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 16:51:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 16:52:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.079761 43.160999 42.987675 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.943848 70.041016 69.898438 +util 98.000000 99.000000 95.000000 +[Apr 27, 2026 - 16:52:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 312 : avg iteration time 49.83 seconds +[Apr 27, 2026 - 16:52:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 312 : data loading time 0.00 +[Apr 27, 2026 - 16:52:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 312 : avg forward pass time 12.64 +[Apr 27, 2026 - 16:52:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 312 : backward pass time 12.01 +[Apr 27, 2026 - 16:52:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 312 : optimizer step time 0.97 +[Apr 27, 2026 - 16:52:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 312-------------------- +[Apr 27, 2026 - 16:52:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0151 iter count: 1.0 +[Apr 27, 2026 - 16:52:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0151 iter count: 1.0 +[Apr 27, 2026 - 16:52:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 16:52:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 16:53:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.079759 43.160999 42.987675 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.924316 70.021484 69.839844 +util 96.500000 99.000000 91.000000 +[Apr 27, 2026 - 16:53:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 313 : avg iteration time 49.23 seconds +[Apr 27, 2026 - 16:53:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 313 : data loading time 0.00 +[Apr 27, 2026 - 16:53:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 313 : avg forward pass time 12.55 +[Apr 27, 2026 - 16:53:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 313 : backward pass time 11.55 +[Apr 27, 2026 - 16:53:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 313 : optimizer step time 0.99 +[Apr 27, 2026 - 16:53:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 313-------------------- +[Apr 27, 2026 - 16:53:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0180 iter count: 1.0 +[Apr 27, 2026 - 16:53:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0180 iter count: 1.0 +[Apr 27, 2026 - 16:53:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 16:53:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 16:53:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.079821 43.160999 42.987675 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.017090 70.054688 69.917969 +util 97.500000 99.000000 93.000000 +[Apr 27, 2026 - 16:53:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 314 : avg iteration time 47.23 seconds +[Apr 27, 2026 - 16:53:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 314 : data loading time 0.00 +[Apr 27, 2026 - 16:53:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 314 : avg forward pass time 11.17 +[Apr 27, 2026 - 16:53:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 314 : backward pass time 11.56 +[Apr 27, 2026 - 16:53:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 314 : optimizer step time 1.21 +[Apr 27, 2026 - 16:53:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 314-------------------- +[Apr 27, 2026 - 16:53:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0158 iter count: 1.0 +[Apr 27, 2026 - 16:53:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0158 iter count: 1.0 +[Apr 27, 2026 - 16:53:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 16:53:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 16:56:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.079818 43.160999 42.987682 +peak_gpu_mem_gb 131.581049 131.620712 131.567828 +peak_gpu_mem_reserved_gb 135.567383 135.599609 135.517578 +util 71.750000 78.000000 64.000000 +[Apr 27, 2026 - 16:56:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 315 : avg iteration time 172.00 seconds +[Apr 27, 2026 - 16:56:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 315 : data loading time 0.00 +[Apr 27, 2026 - 16:56:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 315 : avg forward pass time 77.41 +[Apr 27, 2026 - 16:56:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 315 : backward pass time 8.42 +[Apr 27, 2026 - 16:56:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 315 : optimizer step time 0.33 +[Apr 27, 2026 - 16:56:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 315-------------------- +[Apr 27, 2026 - 16:56:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 278915.1647 iter count: 1.0 +[Apr 27, 2026 - 16:56:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6526 iter count: 1.0 +[Apr 27, 2026 - 16:56:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 278915.1647iter count: 1.0 +[Apr 27, 2026 - 16:56:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 16:56:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.0084iter count: 1.0 +[Apr 27, 2026 - 16:56:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.9335 iter count: 1.0 +[Apr 27, 2026 - 16:56:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.8889 iter count: 1.0 +[Apr 27, 2026 - 16:56:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 464174.8750iter count: 1.0 +[Apr 27, 2026 - 16:56:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 6.4685 iter count: 1.0 +[Apr 27, 2026 - 16:56:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 7114972.0000iter count: 1.0 +[Apr 27, 2026 - 16:56:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6526iter count: 1.0 +[Apr 27, 2026 - 16:56:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 0.9335 iter count: 1.0 +[Apr 27, 2026 - 16:56:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 6.4685 iter count: 1.0 +[Apr 27, 2026 - 16:56:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.0922 iter count: 1.0 +[Apr 27, 2026 - 16:56:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 26488.3848iter count: 1.0 +[Apr 27, 2026 - 16:56:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.3389 iter count: 1.0 +[Apr 27, 2026 - 16:56:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 5871.2681iter count: 1.0 +[Apr 27, 2026 - 16:56:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.4940 iter count: 1.0 +[Apr 27, 2026 - 16:56:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 59155.8242iter count: 1.0 +[Apr 27, 2026 - 16:56:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0174 iter count: 1.0 +[Apr 27, 2026 - 16:56:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 16:56:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 16:57:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.079810 43.160999 42.987675 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.967773 70.058594 69.917969 +util 96.000000 99.000000 91.000000 +[Apr 27, 2026 - 16:57:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 316 : avg iteration time 51.46 seconds +[Apr 27, 2026 - 16:57:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 316 : data loading time 0.00 +[Apr 27, 2026 - 16:57:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 316 : avg forward pass time 13.18 +[Apr 27, 2026 - 16:57:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 316 : backward pass time 11.53 +[Apr 27, 2026 - 16:57:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 316 : optimizer step time 1.03 +[Apr 27, 2026 - 16:57:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 316-------------------- +[Apr 27, 2026 - 16:57:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0163 iter count: 1.0 +[Apr 27, 2026 - 16:57:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0163 iter count: 1.0 +[Apr 27, 2026 - 16:57:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 16:57:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 16:58:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.080062 43.160999 42.987675 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.918945 69.996094 69.843750 +util 95.750000 99.000000 91.000000 +[Apr 27, 2026 - 16:58:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 317 : avg iteration time 48.00 seconds +[Apr 27, 2026 - 16:58:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 317 : data loading time 0.00 +[Apr 27, 2026 - 16:58:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 317 : avg forward pass time 11.35 +[Apr 27, 2026 - 16:58:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 317 : backward pass time 12.58 +[Apr 27, 2026 - 16:58:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 317 : optimizer step time 1.15 +[Apr 27, 2026 - 16:58:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 317-------------------- +[Apr 27, 2026 - 16:58:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0149 iter count: 1.0 +[Apr 27, 2026 - 16:58:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0149 iter count: 1.0 +[Apr 27, 2026 - 16:58:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 16:58:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 16:59:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.080125 43.160999 42.987675 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.948242 70.019531 69.917969 +util 98.000000 99.000000 95.000000 +[Apr 27, 2026 - 16:59:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 318 : avg iteration time 49.70 seconds +[Apr 27, 2026 - 16:59:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 318 : data loading time 0.00 +[Apr 27, 2026 - 16:59:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 318 : avg forward pass time 12.77 +[Apr 27, 2026 - 16:59:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 318 : backward pass time 11.55 +[Apr 27, 2026 - 16:59:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 318 : optimizer step time 1.01 +[Apr 27, 2026 - 16:59:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 318-------------------- +[Apr 27, 2026 - 16:59:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0154 iter count: 1.0 +[Apr 27, 2026 - 16:59:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0154 iter count: 1.0 +[Apr 27, 2026 - 16:59:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 16:59:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 17:00:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.080077 43.160999 42.987675 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.967773 69.996094 69.937500 +util 97.500000 99.000000 93.000000 +[Apr 27, 2026 - 17:00:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 319 : avg iteration time 47.64 seconds +[Apr 27, 2026 - 17:00:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 319 : data loading time 0.00 +[Apr 27, 2026 - 17:00:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 319 : avg forward pass time 11.33 +[Apr 27, 2026 - 17:00:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 319 : backward pass time 11.56 +[Apr 27, 2026 - 17:00:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 319 : optimizer step time 0.98 +[Apr 27, 2026 - 17:00:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 319-------------------- +[Apr 27, 2026 - 17:00:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0151 iter count: 1.0 +[Apr 27, 2026 - 17:00:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0151 iter count: 1.0 +[Apr 27, 2026 - 17:00:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 17:00:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 17:03:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.080068 43.160149 42.987675 +peak_gpu_mem_gb 131.581049 131.620712 131.567828 +peak_gpu_mem_reserved_gb 135.600586 135.658203 135.517578 +util 71.500000 77.000000 65.000000 +[Apr 27, 2026 - 17:03:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 320 : avg iteration time 176.56 seconds +[Apr 27, 2026 - 17:03:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 320 : data loading time 0.00 +[Apr 27, 2026 - 17:03:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 320 : avg forward pass time 79.42 +[Apr 27, 2026 - 17:03:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 320 : backward pass time 8.38 +[Apr 27, 2026 - 17:03:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 320 : optimizer step time 0.34 +[Apr 27, 2026 - 17:03:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 320-------------------- +[Apr 27, 2026 - 17:03:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 519125.2773 iter count: 1.0 +[Apr 27, 2026 - 17:03:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6885 iter count: 1.0 +[Apr 27, 2026 - 17:03:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 519125.2773iter count: 1.0 +[Apr 27, 2026 - 17:03:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 17:03:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.9640iter count: 1.0 +[Apr 27, 2026 - 17:03:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.0334 iter count: 1.0 +[Apr 27, 2026 - 17:03:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.1164 iter count: 1.0 +[Apr 27, 2026 - 17:03:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 630692.5625iter count: 1.0 +[Apr 27, 2026 - 17:03:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 7.8997 iter count: 1.0 +[Apr 27, 2026 - 17:03:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 11215499.0000iter count: 1.0 +[Apr 27, 2026 - 17:03:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6885iter count: 1.0 +[Apr 27, 2026 - 17:03:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 1.0334 iter count: 1.0 +[Apr 27, 2026 - 17:03:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 7.8997 iter count: 1.0 +[Apr 27, 2026 - 17:03:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.9588 iter count: 1.0 +[Apr 27, 2026 - 17:03:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 20283.9141iter count: 1.0 +[Apr 27, 2026 - 17:03:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.0251 iter count: 1.0 +[Apr 27, 2026 - 17:03:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 23158.7734iter count: 1.0 +[Apr 27, 2026 - 17:03:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.8925 iter count: 1.0 +[Apr 27, 2026 - 17:03:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 131275.9688iter count: 1.0 +[Apr 27, 2026 - 17:03:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0191 iter count: 1.0 +[Apr 27, 2026 - 17:03:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 17:03:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 17:03:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.080124 43.160149 42.987675 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.992188 70.035156 69.941406 +util 97.500000 99.000000 94.000000 +[Apr 27, 2026 - 17:03:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 321 : avg iteration time 48.99 seconds +[Apr 27, 2026 - 17:03:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 321 : data loading time 0.00 +[Apr 27, 2026 - 17:03:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 321 : avg forward pass time 12.14 +[Apr 27, 2026 - 17:03:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 321 : backward pass time 11.56 +[Apr 27, 2026 - 17:03:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 321 : optimizer step time 0.98 +[Apr 27, 2026 - 17:03:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 321-------------------- +[Apr 27, 2026 - 17:03:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0156 iter count: 1.0 +[Apr 27, 2026 - 17:03:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0156 iter count: 1.0 +[Apr 27, 2026 - 17:03:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 17:03:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 17:04:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.080191 43.160149 42.987690 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.967773 69.996094 69.937500 +util 98.000000 100.000000 93.000000 +[Apr 27, 2026 - 17:04:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 322 : avg iteration time 53.25 seconds +[Apr 27, 2026 - 17:04:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 322 : data loading time 0.00 +[Apr 27, 2026 - 17:04:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 322 : avg forward pass time 13.93 +[Apr 27, 2026 - 17:04:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 322 : backward pass time 11.94 +[Apr 27, 2026 - 17:04:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 322 : optimizer step time 1.54 +[Apr 27, 2026 - 17:04:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 322-------------------- +[Apr 27, 2026 - 17:04:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0187 iter count: 1.0 +[Apr 27, 2026 - 17:04:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0187 iter count: 1.0 +[Apr 27, 2026 - 17:04:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 17:04:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 17:05:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.080196 43.160149 42.987690 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.006836 70.035156 69.937500 +util 96.750000 100.000000 92.000000 +[Apr 27, 2026 - 17:05:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 323 : avg iteration time 52.53 seconds +[Apr 27, 2026 - 17:05:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 323 : data loading time 0.00 +[Apr 27, 2026 - 17:05:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 323 : avg forward pass time 14.18 +[Apr 27, 2026 - 17:05:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 323 : backward pass time 11.55 +[Apr 27, 2026 - 17:05:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 323 : optimizer step time 1.01 +[Apr 27, 2026 - 17:05:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 323-------------------- +[Apr 27, 2026 - 17:05:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0181 iter count: 1.0 +[Apr 27, 2026 - 17:05:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0181 iter count: 1.0 +[Apr 27, 2026 - 17:05:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 17:05:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 17:06:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.080386 43.160149 42.987679 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.992188 70.035156 69.937500 +util 96.500000 99.000000 92.000000 +[Apr 27, 2026 - 17:06:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 324 : avg iteration time 47.55 seconds +[Apr 27, 2026 - 17:06:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 324 : data loading time 0.00 +[Apr 27, 2026 - 17:06:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 324 : avg forward pass time 11.34 +[Apr 27, 2026 - 17:06:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 324 : backward pass time 11.57 +[Apr 27, 2026 - 17:06:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 324 : optimizer step time 0.97 +[Apr 27, 2026 - 17:06:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 324-------------------- +[Apr 27, 2026 - 17:06:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0150 iter count: 1.0 +[Apr 27, 2026 - 17:06:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0150 iter count: 1.0 +[Apr 27, 2026 - 17:06:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 17:06:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 17:09:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.080401 43.160160 42.987690 +peak_gpu_mem_gb 131.581049 131.620712 131.567828 +peak_gpu_mem_reserved_gb 135.570801 135.599609 135.535156 +util 69.250000 75.000000 64.000000 +[Apr 27, 2026 - 17:09:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 325 : avg iteration time 170.26 seconds +[Apr 27, 2026 - 17:09:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 325 : data loading time 0.00 +[Apr 27, 2026 - 17:09:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 325 : avg forward pass time 76.25 +[Apr 27, 2026 - 17:09:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 325 : backward pass time 8.47 +[Apr 27, 2026 - 17:09:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 325 : optimizer step time 0.34 +[Apr 27, 2026 - 17:09:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 325-------------------- +[Apr 27, 2026 - 17:09:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 208393.1426 iter count: 1.0 +[Apr 27, 2026 - 17:09:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6190 iter count: 1.0 +[Apr 27, 2026 - 17:09:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 208393.1426iter count: 1.0 +[Apr 27, 2026 - 17:09:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 17:09:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.7218iter count: 1.0 +[Apr 27, 2026 - 17:09:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.7568 iter count: 1.0 +[Apr 27, 2026 - 17:09:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.5843 iter count: 1.0 +[Apr 27, 2026 - 17:09:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 390896.6250iter count: 1.0 +[Apr 27, 2026 - 17:09:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 4.5433 iter count: 1.0 +[Apr 27, 2026 - 17:09:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 3869106.7500iter count: 1.0 +[Apr 27, 2026 - 17:09:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6190iter count: 1.0 +[Apr 27, 2026 - 17:09:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.6913 iter count: 1.0 +[Apr 27, 2026 - 17:09:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 1607.6461iter count: 1.0 +[Apr 27, 2026 - 17:09:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.4019 iter count: 1.0 +[Apr 27, 2026 - 17:09:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 6659.2393iter count: 1.0 +[Apr 27, 2026 - 17:09:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.4233 iter count: 1.0 +[Apr 27, 2026 - 17:09:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 51358.9141iter count: 1.0 +[Apr 27, 2026 - 17:09:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.5851 iter count: 1.0 +[Apr 27, 2026 - 17:09:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 9605.7793iter count: 1.0 +[Apr 27, 2026 - 17:09:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0155 iter count: 1.0 +[Apr 27, 2026 - 17:09:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 17:09:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 17:10:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.080778 43.160149 42.987675 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.607422 72.398438 69.917969 +util 95.750000 99.000000 91.000000 +[Apr 27, 2026 - 17:10:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 326 : avg iteration time 50.89 seconds +[Apr 27, 2026 - 17:10:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 326 : data loading time 0.00 +[Apr 27, 2026 - 17:10:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 326 : avg forward pass time 13.25 +[Apr 27, 2026 - 17:10:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 326 : backward pass time 11.81 +[Apr 27, 2026 - 17:10:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 326 : optimizer step time 0.99 +[Apr 27, 2026 - 17:10:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 326-------------------- +[Apr 27, 2026 - 17:10:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0190 iter count: 1.0 +[Apr 27, 2026 - 17:10:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0190 iter count: 1.0 +[Apr 27, 2026 - 17:10:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 17:10:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 17:10:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.080810 43.160149 42.987675 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.573242 72.535156 69.839844 +util 96.250000 100.000000 91.000000 +[Apr 27, 2026 - 17:10:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 327 : avg iteration time 53.52 seconds +[Apr 27, 2026 - 17:10:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 327 : data loading time 0.00 +[Apr 27, 2026 - 17:10:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 327 : avg forward pass time 14.10 +[Apr 27, 2026 - 17:10:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 327 : backward pass time 12.57 +[Apr 27, 2026 - 17:10:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 327 : optimizer step time 1.21 +[Apr 27, 2026 - 17:10:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 327-------------------- +[Apr 27, 2026 - 17:10:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0185 iter count: 1.0 +[Apr 27, 2026 - 17:10:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0185 iter count: 1.0 +[Apr 27, 2026 - 17:10:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 17:10:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 17:11:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.080858 43.160149 42.987682 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.636719 72.437500 69.976562 +util 97.000000 99.000000 93.000000 +[Apr 27, 2026 - 17:11:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 328 : avg iteration time 52.34 seconds +[Apr 27, 2026 - 17:11:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 328 : data loading time 0.00 +[Apr 27, 2026 - 17:11:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 328 : avg forward pass time 14.13 +[Apr 27, 2026 - 17:11:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 328 : backward pass time 11.55 +[Apr 27, 2026 - 17:11:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 328 : optimizer step time 0.99 +[Apr 27, 2026 - 17:11:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 328-------------------- +[Apr 27, 2026 - 17:11:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0190 iter count: 1.0 +[Apr 27, 2026 - 17:11:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0190 iter count: 1.0 +[Apr 27, 2026 - 17:11:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 17:11:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 17:12:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.080853 43.160168 42.987690 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.583008 72.476562 69.898438 +util 97.750000 100.000000 94.000000 +[Apr 27, 2026 - 17:12:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 329 : avg iteration time 49.97 seconds +[Apr 27, 2026 - 17:12:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 329 : data loading time 0.00 +[Apr 27, 2026 - 17:12:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 329 : avg forward pass time 12.40 +[Apr 27, 2026 - 17:12:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 329 : backward pass time 11.54 +[Apr 27, 2026 - 17:12:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 329 : optimizer step time 0.99 +[Apr 27, 2026 - 17:12:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 329-------------------- +[Apr 27, 2026 - 17:12:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0164 iter count: 1.0 +[Apr 27, 2026 - 17:12:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0164 iter count: 1.0 +[Apr 27, 2026 - 17:12:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 17:12:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 17:15:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.081288 43.160149 42.987675 +peak_gpu_mem_gb 131.581049 131.620712 131.567828 +peak_gpu_mem_reserved_gb 135.557617 135.595703 135.458984 +util 72.000000 96.000000 57.000000 +[Apr 27, 2026 - 17:15:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 330 : avg iteration time 181.46 seconds +[Apr 27, 2026 - 17:15:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 330 : data loading time 0.00 +[Apr 27, 2026 - 17:15:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 330 : avg forward pass time 81.58 +[Apr 27, 2026 - 17:15:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 330 : backward pass time 9.67 +[Apr 27, 2026 - 17:15:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 330 : optimizer step time 0.35 +[Apr 27, 2026 - 17:15:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 330-------------------- +[Apr 27, 2026 - 17:15:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 93546.1211 iter count: 1.0 +[Apr 27, 2026 - 17:15:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6699 iter count: 1.0 +[Apr 27, 2026 - 17:15:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 93546.1211iter count: 1.0 +[Apr 27, 2026 - 17:15:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 17:15:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.4264iter count: 1.0 +[Apr 27, 2026 - 17:15:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.3733 iter count: 1.0 +[Apr 27, 2026 - 17:15:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.0512 iter count: 1.0 +[Apr 27, 2026 - 17:15:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 113950.7500iter count: 1.0 +[Apr 27, 2026 - 17:15:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 15.5897iter count: 1.0 +[Apr 27, 2026 - 17:15:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 1332272.8750iter count: 1.0 +[Apr 27, 2026 - 17:15:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6699iter count: 1.0 +[Apr 27, 2026 - 17:15:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.2067 iter count: 1.0 +[Apr 27, 2026 - 17:15:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 4506.6748iter count: 1.0 +[Apr 27, 2026 - 17:15:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 2.4891 iter count: 1.0 +[Apr 27, 2026 - 17:15:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 145.2051iter count: 1.0 +[Apr 27, 2026 - 17:15:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.5404 iter count: 1.0 +[Apr 27, 2026 - 17:15:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 1189.0156iter count: 1.0 +[Apr 27, 2026 - 17:15:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.6346 iter count: 1.0 +[Apr 27, 2026 - 17:15:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 1435.3461iter count: 1.0 +[Apr 27, 2026 - 17:15:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0193 iter count: 1.0 +[Apr 27, 2026 - 17:15:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 17:15:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 17:16:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.081330 43.160149 42.987675 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.002441 70.080078 69.976562 +util 95.500000 100.000000 90.000000 +[Apr 27, 2026 - 17:16:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 331 : avg iteration time 52.35 seconds +[Apr 27, 2026 - 17:16:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 331 : data loading time 0.00 +[Apr 27, 2026 - 17:16:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 331 : avg forward pass time 13.88 +[Apr 27, 2026 - 17:16:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 331 : backward pass time 11.97 +[Apr 27, 2026 - 17:16:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 331 : optimizer step time 0.99 +[Apr 27, 2026 - 17:16:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 331-------------------- +[Apr 27, 2026 - 17:16:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0190 iter count: 1.0 +[Apr 27, 2026 - 17:16:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0190 iter count: 1.0 +[Apr 27, 2026 - 17:16:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 17:16:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 17:17:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.081330 43.160149 42.987675 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.958496 69.982422 69.937500 +util 97.250000 100.000000 92.000000 +[Apr 27, 2026 - 17:17:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 332 : avg iteration time 52.53 seconds +[Apr 27, 2026 - 17:17:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 332 : data loading time 0.00 +[Apr 27, 2026 - 17:17:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 332 : avg forward pass time 14.05 +[Apr 27, 2026 - 17:17:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 332 : backward pass time 11.53 +[Apr 27, 2026 - 17:17:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 332 : optimizer step time 0.98 +[Apr 27, 2026 - 17:17:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 332-------------------- +[Apr 27, 2026 - 17:17:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0254 iter count: 1.0 +[Apr 27, 2026 - 17:17:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0254 iter count: 1.0 +[Apr 27, 2026 - 17:17:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 17:17:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 17:18:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.081307 43.160149 42.987686 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.953613 69.962891 69.937500 +util 97.750000 100.000000 93.000000 +[Apr 27, 2026 - 17:18:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 333 : avg iteration time 47.81 seconds +[Apr 27, 2026 - 17:18:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 333 : data loading time 0.00 +[Apr 27, 2026 - 17:18:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 333 : avg forward pass time 11.27 +[Apr 27, 2026 - 17:18:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 333 : backward pass time 11.56 +[Apr 27, 2026 - 17:18:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 333 : optimizer step time 0.97 +[Apr 27, 2026 - 17:18:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 333-------------------- +[Apr 27, 2026 - 17:18:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0196 iter count: 1.0 +[Apr 27, 2026 - 17:18:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0196 iter count: 1.0 +[Apr 27, 2026 - 17:18:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 17:18:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 17:19:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.081294 43.160149 42.987698 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.958496 69.962891 69.957031 +util 96.750000 100.000000 92.000000 +[Apr 27, 2026 - 17:19:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 334 : avg iteration time 54.71 seconds +[Apr 27, 2026 - 17:19:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 334 : data loading time 0.00 +[Apr 27, 2026 - 17:19:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 334 : avg forward pass time 14.35 +[Apr 27, 2026 - 17:19:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 334 : backward pass time 12.86 +[Apr 27, 2026 - 17:19:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 334 : optimizer step time 1.59 +[Apr 27, 2026 - 17:19:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 334-------------------- +[Apr 27, 2026 - 17:19:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0245 iter count: 1.0 +[Apr 27, 2026 - 17:19:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0245 iter count: 1.0 +[Apr 27, 2026 - 17:19:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 17:19:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 17:22:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.081358 43.160152 42.987675 +peak_gpu_mem_gb 131.581049 131.620712 131.567828 +peak_gpu_mem_reserved_gb 135.621094 135.673828 135.498047 +util 71.750000 79.000000 65.000000 +[Apr 27, 2026 - 17:22:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 335 : avg iteration time 175.96 seconds +[Apr 27, 2026 - 17:22:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 335 : data loading time 0.00 +[Apr 27, 2026 - 17:22:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 335 : avg forward pass time 79.38 +[Apr 27, 2026 - 17:22:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 335 : backward pass time 8.44 +[Apr 27, 2026 - 17:22:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 335 : optimizer step time 0.34 +[Apr 27, 2026 - 17:22:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 335-------------------- +[Apr 27, 2026 - 17:22:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 240290.4355 iter count: 1.0 +[Apr 27, 2026 - 17:22:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6880 iter count: 1.0 +[Apr 27, 2026 - 17:22:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 240290.4355iter count: 1.0 +[Apr 27, 2026 - 17:22:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 17:22:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.4937iter count: 1.0 +[Apr 27, 2026 - 17:22:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.9045 iter count: 1.0 +[Apr 27, 2026 - 17:22:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.4108 iter count: 1.0 +[Apr 27, 2026 - 17:22:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 289849.9375iter count: 1.0 +[Apr 27, 2026 - 17:22:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 6.1049 iter count: 1.0 +[Apr 27, 2026 - 17:22:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 2735117.2500iter count: 1.0 +[Apr 27, 2026 - 17:22:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6880iter count: 1.0 +[Apr 27, 2026 - 17:22:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.2501 iter count: 1.0 +[Apr 27, 2026 - 17:22:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 36320.3516iter count: 1.0 +[Apr 27, 2026 - 17:22:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.0515 iter count: 1.0 +[Apr 27, 2026 - 17:22:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 3304.6899iter count: 1.0 +[Apr 27, 2026 - 17:22:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.9989 iter count: 1.0 +[Apr 27, 2026 - 17:22:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 2974.6648iter count: 1.0 +[Apr 27, 2026 - 17:22:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.6872 iter count: 1.0 +[Apr 27, 2026 - 17:22:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 11781.7578iter count: 1.0 +[Apr 27, 2026 - 17:22:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0244 iter count: 1.0 +[Apr 27, 2026 - 17:22:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 17:22:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 17:22:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.081362 43.160164 42.987675 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.958496 69.982422 69.937500 +util 96.500000 99.000000 92.000000 +[Apr 27, 2026 - 17:22:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 336 : avg iteration time 52.37 seconds +[Apr 27, 2026 - 17:22:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 336 : data loading time 0.00 +[Apr 27, 2026 - 17:22:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 336 : avg forward pass time 13.61 +[Apr 27, 2026 - 17:22:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 336 : backward pass time 11.56 +[Apr 27, 2026 - 17:22:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 336 : optimizer step time 0.97 +[Apr 27, 2026 - 17:22:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 336-------------------- +[Apr 27, 2026 - 17:22:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0205 iter count: 1.0 +[Apr 27, 2026 - 17:22:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0205 iter count: 1.0 +[Apr 27, 2026 - 17:22:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 17:22:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 17:23:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.081560 43.160156 42.987679 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.002441 70.080078 69.937500 +util 94.250000 99.000000 88.000000 +[Apr 27, 2026 - 17:23:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 337 : avg iteration time 47.78 seconds +[Apr 27, 2026 - 17:23:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 337 : data loading time 0.00 +[Apr 27, 2026 - 17:23:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 337 : avg forward pass time 11.39 +[Apr 27, 2026 - 17:23:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 337 : backward pass time 11.55 +[Apr 27, 2026 - 17:23:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 337 : optimizer step time 1.01 +[Apr 27, 2026 - 17:23:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 337-------------------- +[Apr 27, 2026 - 17:23:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0190 iter count: 1.0 +[Apr 27, 2026 - 17:23:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0190 iter count: 1.0 +[Apr 27, 2026 - 17:23:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 17:23:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 17:24:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.081697 43.160156 42.987679 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.963379 69.996094 69.923828 +util 98.250000 99.000000 96.000000 +[Apr 27, 2026 - 17:24:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 338 : avg iteration time 54.15 seconds +[Apr 27, 2026 - 17:24:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 338 : data loading time 0.00 +[Apr 27, 2026 - 17:24:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 338 : avg forward pass time 14.30 +[Apr 27, 2026 - 17:24:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 338 : backward pass time 12.89 +[Apr 27, 2026 - 17:24:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 338 : optimizer step time 1.09 +[Apr 27, 2026 - 17:24:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 338-------------------- +[Apr 27, 2026 - 17:24:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0203 iter count: 1.0 +[Apr 27, 2026 - 17:24:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0203 iter count: 1.0 +[Apr 27, 2026 - 17:24:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 17:24:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 17:25:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.081705 43.160156 42.987679 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.968262 69.996094 69.917969 +util 96.500000 99.000000 92.000000 +[Apr 27, 2026 - 17:25:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 339 : avg iteration time 50.07 seconds +[Apr 27, 2026 - 17:25:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 339 : data loading time 0.00 +[Apr 27, 2026 - 17:25:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 339 : avg forward pass time 12.97 +[Apr 27, 2026 - 17:25:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 339 : backward pass time 11.56 +[Apr 27, 2026 - 17:25:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 339 : optimizer step time 1.01 +[Apr 27, 2026 - 17:25:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 339-------------------- +[Apr 27, 2026 - 17:25:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0173 iter count: 1.0 +[Apr 27, 2026 - 17:25:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0173 iter count: 1.0 +[Apr 27, 2026 - 17:25:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 17:25:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 17:28:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.081321 43.160175 42.987679 +peak_gpu_mem_gb 131.581049 131.620712 131.567828 +peak_gpu_mem_reserved_gb 135.648926 135.710938 135.535156 +util 71.000000 78.000000 64.000000 +[Apr 27, 2026 - 17:28:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 340 : avg iteration time 172.41 seconds +[Apr 27, 2026 - 17:28:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 340 : data loading time 0.00 +[Apr 27, 2026 - 17:28:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 340 : avg forward pass time 77.67 +[Apr 27, 2026 - 17:28:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 340 : backward pass time 8.41 +[Apr 27, 2026 - 17:28:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 340 : optimizer step time 0.33 +[Apr 27, 2026 - 17:28:21 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 340-------------------- +[Apr 27, 2026 - 17:28:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 74793.8145 iter count: 1.0 +[Apr 27, 2026 - 17:28:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5853 iter count: 1.0 +[Apr 27, 2026 - 17:28:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 74793.8145iter count: 1.0 +[Apr 27, 2026 - 17:28:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 17:28:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.8282iter count: 1.0 +[Apr 27, 2026 - 17:28:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 2.4599 iter count: 1.0 +[Apr 27, 2026 - 17:28:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.0031 iter count: 1.0 +[Apr 27, 2026 - 17:28:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 129297.9609iter count: 1.0 +[Apr 27, 2026 - 17:28:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 136.9848iter count: 1.0 +[Apr 27, 2026 - 17:28:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 1210009.6250iter count: 1.0 +[Apr 27, 2026 - 17:28:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5853iter count: 1.0 +[Apr 27, 2026 - 17:28:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.4965 iter count: 1.0 +[Apr 27, 2026 - 17:28:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 59451.2148iter count: 1.0 +[Apr 27, 2026 - 17:28:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.8089 iter count: 1.0 +[Apr 27, 2026 - 17:28:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 111054.2422iter count: 1.0 +[Apr 27, 2026 - 17:28:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.1536 iter count: 1.0 +[Apr 27, 2026 - 17:28:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 548.4774iter count: 1.0 +[Apr 27, 2026 - 17:28:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.2064 iter count: 1.0 +[Apr 27, 2026 - 17:28:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 4504.5005iter count: 1.0 +[Apr 27, 2026 - 17:28:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0182 iter count: 1.0 +[Apr 27, 2026 - 17:28:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 17:28:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 17:29:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.081336 43.160156 42.987717 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.041504 70.093750 69.957031 +util 96.500000 99.000000 91.000000 +[Apr 27, 2026 - 17:29:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 341 : avg iteration time 49.53 seconds +[Apr 27, 2026 - 17:29:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 341 : data loading time 0.00 +[Apr 27, 2026 - 17:29:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 341 : avg forward pass time 12.11 +[Apr 27, 2026 - 17:29:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 341 : backward pass time 11.54 +[Apr 27, 2026 - 17:29:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 341 : optimizer step time 0.99 +[Apr 27, 2026 - 17:29:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 341-------------------- +[Apr 27, 2026 - 17:29:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0153 iter count: 1.0 +[Apr 27, 2026 - 17:29:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0153 iter count: 1.0 +[Apr 27, 2026 - 17:29:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 17:29:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 17:29:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.081610 43.160156 42.987679 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.007324 70.093750 69.904297 +util 96.500000 99.000000 91.000000 +[Apr 27, 2026 - 17:29:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 342 : avg iteration time 47.80 seconds +[Apr 27, 2026 - 17:29:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 342 : data loading time 0.00 +[Apr 27, 2026 - 17:29:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 342 : avg forward pass time 11.15 +[Apr 27, 2026 - 17:29:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 342 : backward pass time 11.56 +[Apr 27, 2026 - 17:29:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 342 : optimizer step time 0.98 +[Apr 27, 2026 - 17:29:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 342-------------------- +[Apr 27, 2026 - 17:29:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0175 iter count: 1.0 +[Apr 27, 2026 - 17:29:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0175 iter count: 1.0 +[Apr 27, 2026 - 17:29:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 17:29:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 17:30:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.081610 43.160156 42.987679 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.992676 70.074219 69.884766 +util 98.500000 100.000000 95.000000 +[Apr 27, 2026 - 17:30:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 343 : avg iteration time 51.03 seconds +[Apr 27, 2026 - 17:30:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 343 : data loading time 0.00 +[Apr 27, 2026 - 17:30:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 343 : avg forward pass time 12.89 +[Apr 27, 2026 - 17:30:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 343 : backward pass time 12.11 +[Apr 27, 2026 - 17:30:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 343 : optimizer step time 1.56 +[Apr 27, 2026 - 17:30:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 343-------------------- +[Apr 27, 2026 - 17:30:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0166 iter count: 1.0 +[Apr 27, 2026 - 17:30:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0166 iter count: 1.0 +[Apr 27, 2026 - 17:30:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 17:30:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 17:31:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.081669 43.160156 42.987679 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.002441 70.093750 69.884766 +util 96.250000 99.000000 92.000000 +[Apr 27, 2026 - 17:31:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 344 : avg iteration time 52.62 seconds +[Apr 27, 2026 - 17:31:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 344 : data loading time 0.00 +[Apr 27, 2026 - 17:31:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 344 : avg forward pass time 14.24 +[Apr 27, 2026 - 17:31:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 344 : backward pass time 11.57 +[Apr 27, 2026 - 17:31:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 344 : optimizer step time 0.98 +[Apr 27, 2026 - 17:31:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 344-------------------- +[Apr 27, 2026 - 17:31:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0155 iter count: 1.0 +[Apr 27, 2026 - 17:31:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0155 iter count: 1.0 +[Apr 27, 2026 - 17:31:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 17:31:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 17:34:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.081802 43.160156 42.987701 +peak_gpu_mem_gb 131.581049 131.620712 131.567828 +peak_gpu_mem_reserved_gb 135.576660 135.619141 135.535156 +util 68.750000 74.000000 63.000000 +[Apr 27, 2026 - 17:34:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 345 : avg iteration time 172.10 seconds +[Apr 27, 2026 - 17:34:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 345 : data loading time 0.00 +[Apr 27, 2026 - 17:34:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 345 : avg forward pass time 77.51 +[Apr 27, 2026 - 17:34:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 345 : backward pass time 8.41 +[Apr 27, 2026 - 17:34:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 345 : optimizer step time 0.34 +[Apr 27, 2026 - 17:34:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 345-------------------- +[Apr 27, 2026 - 17:34:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 466365.0234 iter count: 1.0 +[Apr 27, 2026 - 17:34:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5920 iter count: 1.0 +[Apr 27, 2026 - 17:34:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 466365.0234iter count: 1.0 +[Apr 27, 2026 - 17:34:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 17:34:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.5730iter count: 1.0 +[Apr 27, 2026 - 17:34:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.5057 iter count: 1.0 +[Apr 27, 2026 - 17:34:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.0948 iter count: 1.0 +[Apr 27, 2026 - 17:34:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 923070.1250iter count: 1.0 +[Apr 27, 2026 - 17:34:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 2.7493 iter count: 1.0 +[Apr 27, 2026 - 17:34:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 10740511.0000iter count: 1.0 +[Apr 27, 2026 - 17:34:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5920iter count: 1.0 +[Apr 27, 2026 - 17:34:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 1.4115 iter count: 1.0 +[Apr 27, 2026 - 17:34:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 16.8277 iter count: 1.0 +[Apr 27, 2026 - 17:34:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.2717 iter count: 1.0 +[Apr 27, 2026 - 17:34:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 37923.5039iter count: 1.0 +[Apr 27, 2026 - 17:34:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.4322 iter count: 1.0 +[Apr 27, 2026 - 17:34:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 957.5208iter count: 1.0 +[Apr 27, 2026 - 17:34:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 0.8258 iter count: 1.0 +[Apr 27, 2026 - 17:34:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 5.2150 iter count: 1.0 +[Apr 27, 2026 - 17:34:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0159 iter count: 1.0 +[Apr 27, 2026 - 17:34:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 17:34:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 17:35:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.081872 43.160156 42.987679 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.036621 70.158203 69.917969 +util 96.000000 99.000000 91.000000 +[Apr 27, 2026 - 17:35:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 346 : avg iteration time 54.69 seconds +[Apr 27, 2026 - 17:35:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 346 : data loading time 0.00 +[Apr 27, 2026 - 17:35:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 346 : avg forward pass time 14.59 +[Apr 27, 2026 - 17:35:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 346 : backward pass time 11.56 +[Apr 27, 2026 - 17:35:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 346 : optimizer step time 0.97 +[Apr 27, 2026 - 17:35:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 346-------------------- +[Apr 27, 2026 - 17:35:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0232 iter count: 1.0 +[Apr 27, 2026 - 17:35:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0232 iter count: 1.0 +[Apr 27, 2026 - 17:35:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 17:35:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 17:36:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.081874 43.160156 42.987686 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.017090 70.080078 69.878906 +util 98.250000 100.000000 95.000000 +[Apr 27, 2026 - 17:36:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 347 : avg iteration time 51.19 seconds +[Apr 27, 2026 - 17:36:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 347 : data loading time 0.00 +[Apr 27, 2026 - 17:36:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 347 : avg forward pass time 12.88 +[Apr 27, 2026 - 17:36:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 347 : backward pass time 12.27 +[Apr 27, 2026 - 17:36:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 347 : optimizer step time 1.58 +[Apr 27, 2026 - 17:36:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 347-------------------- +[Apr 27, 2026 - 17:36:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0225 iter count: 1.0 +[Apr 27, 2026 - 17:36:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0225 iter count: 1.0 +[Apr 27, 2026 - 17:36:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 17:36:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 17:37:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.081878 43.160168 42.987679 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.943848 70.099609 69.878906 +util 97.250000 100.000000 92.000000 +[Apr 27, 2026 - 17:37:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 348 : avg iteration time 52.77 seconds +[Apr 27, 2026 - 17:37:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 348 : data loading time 0.00 +[Apr 27, 2026 - 17:37:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 348 : avg forward pass time 14.29 +[Apr 27, 2026 - 17:37:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 348 : backward pass time 11.64 +[Apr 27, 2026 - 17:37:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 348 : optimizer step time 0.98 +[Apr 27, 2026 - 17:37:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 348-------------------- +[Apr 27, 2026 - 17:37:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0212 iter count: 1.0 +[Apr 27, 2026 - 17:37:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0212 iter count: 1.0 +[Apr 27, 2026 - 17:37:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 17:37:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 17:38:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.081936 43.160156 42.987679 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.963379 70.099609 69.878906 +util 96.250000 100.000000 91.000000 +[Apr 27, 2026 - 17:38:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 349 : avg iteration time 47.08 seconds +[Apr 27, 2026 - 17:38:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 349 : data loading time 0.00 +[Apr 27, 2026 - 17:38:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 349 : avg forward pass time 11.36 +[Apr 27, 2026 - 17:38:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 349 : backward pass time 11.57 +[Apr 27, 2026 - 17:38:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 349 : optimizer step time 0.99 +[Apr 27, 2026 - 17:38:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 349-------------------- +[Apr 27, 2026 - 17:38:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0186 iter count: 1.0 +[Apr 27, 2026 - 17:38:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0186 iter count: 1.0 +[Apr 27, 2026 - 17:38:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 17:38:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 17:40:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.081791 43.160156 42.987679 +peak_gpu_mem_gb 131.581049 131.620712 131.567828 +peak_gpu_mem_reserved_gb 135.561523 135.619141 135.478516 +util 68.750000 74.000000 63.000000 +[Apr 27, 2026 - 17:40:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 350 : avg iteration time 172.46 seconds +[Apr 27, 2026 - 17:40:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 350 : data loading time 0.00 +[Apr 27, 2026 - 17:40:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 350 : avg forward pass time 77.68 +[Apr 27, 2026 - 17:40:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 350 : backward pass time 8.40 +[Apr 27, 2026 - 17:40:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 350 : optimizer step time 0.34 +[Apr 27, 2026 - 17:40:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 350-------------------- +[Apr 27, 2026 - 17:40:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 545838.2051 iter count: 1.0 +[Apr 27, 2026 - 17:40:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6474 iter count: 1.0 +[Apr 27, 2026 - 17:40:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 545838.2051iter count: 1.0 +[Apr 27, 2026 - 17:40:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 17:40:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.5477iter count: 1.0 +[Apr 27, 2026 - 17:40:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.7979 iter count: 1.0 +[Apr 27, 2026 - 17:40:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.5817 iter count: 1.0 +[Apr 27, 2026 - 17:40:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 1017538.0625iter count: 1.0 +[Apr 27, 2026 - 17:40:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 4.9318 iter count: 1.0 +[Apr 27, 2026 - 17:40:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 28444884.0000iter count: 1.0 +[Apr 27, 2026 - 17:40:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6474iter count: 1.0 +[Apr 27, 2026 - 17:40:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.8220 iter count: 1.0 +[Apr 27, 2026 - 17:40:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 2087.8984iter count: 1.0 +[Apr 27, 2026 - 17:40:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.5381 iter count: 1.0 +[Apr 27, 2026 - 17:40:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 64612.6953iter count: 1.0 +[Apr 27, 2026 - 17:40:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 2.2783 iter count: 1.0 +[Apr 27, 2026 - 17:40:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 95.2620 iter count: 1.0 +[Apr 27, 2026 - 17:40:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.6372 iter count: 1.0 +[Apr 27, 2026 - 17:40:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 78771.7031iter count: 1.0 +[Apr 27, 2026 - 17:40:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0205 iter count: 1.0 +[Apr 27, 2026 - 17:40:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 17:40:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 17:41:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.081846 43.160156 42.987679 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.943359 70.015625 69.917969 +util 98.250000 100.000000 93.000000 +[Apr 27, 2026 - 17:41:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 351 : avg iteration time 51.68 seconds +[Apr 27, 2026 - 17:41:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 351 : data loading time 0.00 +[Apr 27, 2026 - 17:41:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 351 : avg forward pass time 13.41 +[Apr 27, 2026 - 17:41:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 351 : backward pass time 11.59 +[Apr 27, 2026 - 17:41:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 351 : optimizer step time 1.35 +[Apr 27, 2026 - 17:41:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 351-------------------- +[Apr 27, 2026 - 17:41:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0168 iter count: 1.0 +[Apr 27, 2026 - 17:41:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0168 iter count: 1.0 +[Apr 27, 2026 - 17:41:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 17:41:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 17:42:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.081913 43.160179 42.987679 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.982422 70.093750 69.898438 +util 96.500000 99.000000 91.000000 +[Apr 27, 2026 - 17:42:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 352 : avg iteration time 50.37 seconds +[Apr 27, 2026 - 17:42:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 352 : data loading time 0.00 +[Apr 27, 2026 - 17:42:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 352 : avg forward pass time 12.57 +[Apr 27, 2026 - 17:42:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 352 : backward pass time 12.64 +[Apr 27, 2026 - 17:42:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 352 : optimizer step time 1.02 +[Apr 27, 2026 - 17:42:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 352-------------------- +[Apr 27, 2026 - 17:42:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0160 iter count: 1.0 +[Apr 27, 2026 - 17:42:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0160 iter count: 1.0 +[Apr 27, 2026 - 17:42:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 17:42:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 17:43:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.081963 43.160156 42.987679 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.860352 69.921875 69.820312 +util 95.500000 99.000000 90.000000 +[Apr 27, 2026 - 17:43:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 353 : avg iteration time 51.73 seconds +[Apr 27, 2026 - 17:43:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 353 : data loading time 0.00 +[Apr 27, 2026 - 17:43:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 353 : avg forward pass time 13.81 +[Apr 27, 2026 - 17:43:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 353 : backward pass time 11.57 +[Apr 27, 2026 - 17:43:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 353 : optimizer step time 0.98 +[Apr 27, 2026 - 17:43:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 353-------------------- +[Apr 27, 2026 - 17:43:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0175 iter count: 1.0 +[Apr 27, 2026 - 17:43:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0175 iter count: 1.0 +[Apr 27, 2026 - 17:43:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 17:43:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 17:44:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.081962 43.160156 42.987679 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.987305 70.015625 69.957031 +util 97.750000 100.000000 94.000000 +[Apr 27, 2026 - 17:44:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 354 : avg iteration time 47.44 seconds +[Apr 27, 2026 - 17:44:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 354 : data loading time 0.00 +[Apr 27, 2026 - 17:44:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 354 : avg forward pass time 11.21 +[Apr 27, 2026 - 17:44:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 354 : backward pass time 11.53 +[Apr 27, 2026 - 17:44:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 354 : optimizer step time 1.02 +[Apr 27, 2026 - 17:44:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 354-------------------- +[Apr 27, 2026 - 17:44:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0173 iter count: 1.0 +[Apr 27, 2026 - 17:44:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0173 iter count: 1.0 +[Apr 27, 2026 - 17:44:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 17:44:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 17:47:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.082049 43.160168 42.987679 +peak_gpu_mem_gb 131.581049 131.620712 131.567828 +peak_gpu_mem_reserved_gb 135.513184 135.560547 135.496094 +util 71.000000 78.000000 64.000000 +[Apr 27, 2026 - 17:47:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 355 : avg iteration time 174.51 seconds +[Apr 27, 2026 - 17:47:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 355 : data loading time 0.00 +[Apr 27, 2026 - 17:47:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 355 : avg forward pass time 78.39 +[Apr 27, 2026 - 17:47:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 355 : backward pass time 8.39 +[Apr 27, 2026 - 17:47:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 355 : optimizer step time 0.34 +[Apr 27, 2026 - 17:47:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 355-------------------- +[Apr 27, 2026 - 17:47:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 156902.6641 iter count: 1.0 +[Apr 27, 2026 - 17:47:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5660 iter count: 1.0 +[Apr 27, 2026 - 17:47:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 156902.6641iter count: 1.0 +[Apr 27, 2026 - 17:47:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 17:47:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.8691iter count: 1.0 +[Apr 27, 2026 - 17:47:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.5610 iter count: 1.0 +[Apr 27, 2026 - 17:47:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.6283 iter count: 1.0 +[Apr 27, 2026 - 17:47:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 315189.5000iter count: 1.0 +[Apr 27, 2026 - 17:47:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 22.6911iter count: 1.0 +[Apr 27, 2026 - 17:47:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 4224940.0000iter count: 1.0 +[Apr 27, 2026 - 17:47:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5660iter count: 1.0 +[Apr 27, 2026 - 17:47:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 6.2148 iter count: 1.0 +[Apr 27, 2026 - 17:47:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 250090.9062iter count: 1.0 +[Apr 27, 2026 - 17:47:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.5624 iter count: 1.0 +[Apr 27, 2026 - 17:47:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 1242.3282iter count: 1.0 +[Apr 27, 2026 - 17:47:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.7715 iter count: 1.0 +[Apr 27, 2026 - 17:47:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 1887.4358iter count: 1.0 +[Apr 27, 2026 - 17:47:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.1208 iter count: 1.0 +[Apr 27, 2026 - 17:47:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 207230.0469iter count: 1.0 +[Apr 27, 2026 - 17:47:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0192 iter count: 1.0 +[Apr 27, 2026 - 17:47:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 17:47:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 17:48:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.082047 43.160156 42.987679 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.592773 72.460938 69.937500 +util 98.000000 100.000000 94.000000 +[Apr 27, 2026 - 17:48:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 356 : avg iteration time 54.97 seconds +[Apr 27, 2026 - 17:48:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 356 : data loading time 0.00 +[Apr 27, 2026 - 17:48:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 356 : avg forward pass time 13.36 +[Apr 27, 2026 - 17:48:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 356 : backward pass time 14.38 +[Apr 27, 2026 - 17:48:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 356 : optimizer step time 2.22 +[Apr 27, 2026 - 17:48:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 356-------------------- +[Apr 27, 2026 - 17:48:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0205 iter count: 1.0 +[Apr 27, 2026 - 17:48:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0205 iter count: 1.0 +[Apr 27, 2026 - 17:48:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 17:48:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.05 seconds +[Apr 27, 2026 - 17:49:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.081919 43.160156 42.987679 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.612305 72.480469 69.937500 +util 93.000000 99.000000 85.000000 +[Apr 27, 2026 - 17:49:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 357 : avg iteration time 76.79 seconds +[Apr 27, 2026 - 17:49:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 357 : data loading time 0.00 +[Apr 27, 2026 - 17:49:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 357 : avg forward pass time 18.56 +[Apr 27, 2026 - 17:49:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 357 : backward pass time 18.19 +[Apr 27, 2026 - 17:49:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 357 : optimizer step time 2.61 +[Apr 27, 2026 - 17:49:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 357-------------------- +[Apr 27, 2026 - 17:49:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0160 iter count: 1.0 +[Apr 27, 2026 - 17:49:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0160 iter count: 1.0 +[Apr 27, 2026 - 17:49:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 17:49:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 17:50:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.081917 43.160156 42.987679 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.583008 72.480469 69.937500 +util 98.500000 100.000000 94.000000 +[Apr 27, 2026 - 17:50:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 358 : avg iteration time 75.60 seconds +[Apr 27, 2026 - 17:50:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 358 : data loading time 0.00 +[Apr 27, 2026 - 17:50:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 358 : avg forward pass time 17.85 +[Apr 27, 2026 - 17:50:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 358 : backward pass time 19.10 +[Apr 27, 2026 - 17:50:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 358 : optimizer step time 2.21 +[Apr 27, 2026 - 17:50:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 358-------------------- +[Apr 27, 2026 - 17:50:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0157 iter count: 1.0 +[Apr 27, 2026 - 17:50:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0157 iter count: 1.0 +[Apr 27, 2026 - 17:50:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 17:50:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.05 seconds +[Apr 27, 2026 - 17:51:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.081923 43.160156 42.987679 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.602539 72.480469 69.937500 +util 96.500000 100.000000 92.000000 +[Apr 27, 2026 - 17:51:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 359 : avg iteration time 76.09 seconds +[Apr 27, 2026 - 17:51:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 359 : data loading time 0.00 +[Apr 27, 2026 - 17:51:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 359 : avg forward pass time 18.55 +[Apr 27, 2026 - 17:51:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 359 : backward pass time 18.28 +[Apr 27, 2026 - 17:51:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 359 : optimizer step time 2.38 +[Apr 27, 2026 - 17:51:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 359-------------------- +[Apr 27, 2026 - 17:51:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0180 iter count: 1.0 +[Apr 27, 2026 - 17:51:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0180 iter count: 1.0 +[Apr 27, 2026 - 17:51:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 17:51:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 17:56:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.082698 43.163055 42.987679 +peak_gpu_mem_gb 129.581033 131.620712 123.567761 +peak_gpu_mem_reserved_gb 133.555176 135.634766 127.468750 +util 81.500000 86.000000 74.000000 +[Apr 27, 2026 - 17:56:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 360 : avg iteration time 307.49 seconds +[Apr 27, 2026 - 17:56:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 360 : data loading time 0.00 +[Apr 27, 2026 - 17:56:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 360 : avg forward pass time 138.47 +[Apr 27, 2026 - 17:56:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 360 : backward pass time 15.25 +[Apr 27, 2026 - 17:56:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 360 : optimizer step time 0.34 +[Apr 27, 2026 - 17:56:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 360-------------------- +[Apr 27, 2026 - 17:56:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 265173.9709 iter count: 1.0 +[Apr 27, 2026 - 17:56:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5314 iter count: 1.0 +[Apr 27, 2026 - 17:56:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 265173.9709iter count: 1.0 +[Apr 27, 2026 - 17:56:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 17:56:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.5556iter count: 1.0 +[Apr 27, 2026 - 17:56:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.5569 iter count: 1.0 +[Apr 27, 2026 - 17:56:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.8983 iter count: 1.0 +[Apr 27, 2026 - 17:56:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 348888.2812iter count: 1.0 +[Apr 27, 2026 - 17:56:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 3.0462 iter count: 1.0 +[Apr 27, 2026 - 17:56:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 7251228.0000iter count: 1.0 +[Apr 27, 2026 - 17:56:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5314iter count: 1.0 +[Apr 27, 2026 - 17:56:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 7.8983 iter count: 1.0 +[Apr 27, 2026 - 17:56:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 7251228.0000iter count: 1.0 +[Apr 27, 2026 - 17:56:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.8786 iter count: 1.0 +[Apr 27, 2026 - 17:56:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 2338.1479iter count: 1.0 +[Apr 27, 2026 - 17:56:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 7.1944 iter count: 1.0 +[Apr 27, 2026 - 17:56:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 1774178.8750iter count: 1.0 +[Apr 27, 2026 - 17:56:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.2812 iter count: 1.0 +[Apr 27, 2026 - 17:56:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 285590.7188iter count: 1.0 +[Apr 27, 2026 - 17:56:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0164 iter count: 1.0 +[Apr 27, 2026 - 17:56:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 17:56:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 17:58:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.082823 43.163055 42.987701 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.055664 70.078125 70.035156 +util 97.750000 100.000000 93.000000 +[Apr 27, 2026 - 17:58:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 361 : avg iteration time 82.79 seconds +[Apr 27, 2026 - 17:58:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 361 : data loading time 0.00 +[Apr 27, 2026 - 17:58:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 361 : avg forward pass time 21.80 +[Apr 27, 2026 - 17:58:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 361 : backward pass time 18.31 +[Apr 27, 2026 - 17:58:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 361 : optimizer step time 2.35 +[Apr 27, 2026 - 17:58:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 361-------------------- +[Apr 27, 2026 - 17:58:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0176 iter count: 1.0 +[Apr 27, 2026 - 17:58:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0176 iter count: 1.0 +[Apr 27, 2026 - 17:58:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 17:58:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 17:59:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.082834 43.163078 42.987701 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.021484 70.117188 69.957031 +util 98.000000 100.000000 94.000000 +[Apr 27, 2026 - 17:59:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 362 : avg iteration time 76.55 seconds +[Apr 27, 2026 - 17:59:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 362 : data loading time 0.00 +[Apr 27, 2026 - 17:59:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 362 : avg forward pass time 18.31 +[Apr 27, 2026 - 17:59:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 362 : backward pass time 18.36 +[Apr 27, 2026 - 17:59:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 362 : optimizer step time 2.41 +[Apr 27, 2026 - 17:59:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 362-------------------- +[Apr 27, 2026 - 17:59:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0154 iter count: 1.0 +[Apr 27, 2026 - 17:59:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0154 iter count: 1.0 +[Apr 27, 2026 - 17:59:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 17:59:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 18:01:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.082823 43.163055 42.987679 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.006836 70.078125 69.976562 +util 96.500000 100.000000 92.000000 +[Apr 27, 2026 - 18:01:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 363 : avg iteration time 81.89 seconds +[Apr 27, 2026 - 18:01:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 363 : data loading time 0.00 +[Apr 27, 2026 - 18:01:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 363 : avg forward pass time 20.96 +[Apr 27, 2026 - 18:01:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 363 : backward pass time 18.36 +[Apr 27, 2026 - 18:01:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 363 : optimizer step time 2.34 +[Apr 27, 2026 - 18:01:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 363-------------------- +[Apr 27, 2026 - 18:01:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0162 iter count: 1.0 +[Apr 27, 2026 - 18:01:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0162 iter count: 1.0 +[Apr 27, 2026 - 18:01:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 18:01:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.05 seconds +[Apr 27, 2026 - 18:02:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.082930 43.163055 42.987701 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.031250 70.195312 69.898438 +util 97.250000 100.000000 92.000000 +[Apr 27, 2026 - 18:02:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 364 : avg iteration time 76.62 seconds +[Apr 27, 2026 - 18:02:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 364 : data loading time 0.00 +[Apr 27, 2026 - 18:02:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 364 : avg forward pass time 18.37 +[Apr 27, 2026 - 18:02:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 364 : backward pass time 18.36 +[Apr 27, 2026 - 18:02:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 364 : optimizer step time 2.36 +[Apr 27, 2026 - 18:02:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 364-------------------- +[Apr 27, 2026 - 18:02:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0162 iter count: 1.0 +[Apr 27, 2026 - 18:02:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0162 iter count: 1.0 +[Apr 27, 2026 - 18:02:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 18:02:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.05 seconds +[Apr 27, 2026 - 18:06:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.082830 43.163078 42.987701 +peak_gpu_mem_gb 129.581032 131.620712 123.567759 +peak_gpu_mem_reserved_gb 133.574707 135.599609 127.546875 +util 70.000000 77.000000 60.000000 +[Apr 27, 2026 - 18:06:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 365 : avg iteration time 275.38 seconds +[Apr 27, 2026 - 18:06:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 365 : data loading time 0.00 +[Apr 27, 2026 - 18:06:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 365 : avg forward pass time 123.39 +[Apr 27, 2026 - 18:06:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 365 : backward pass time 14.49 +[Apr 27, 2026 - 18:06:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 365 : optimizer step time 0.37 +[Apr 27, 2026 - 18:06:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 365-------------------- +[Apr 27, 2026 - 18:06:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 440398.5742 iter count: 1.0 +[Apr 27, 2026 - 18:06:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.7012 iter count: 1.0 +[Apr 27, 2026 - 18:06:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 440398.5742iter count: 1.0 +[Apr 27, 2026 - 18:06:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 18:06:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 5.1175iter count: 1.0 +[Apr 27, 2026 - 18:06:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.3403 iter count: 1.0 +[Apr 27, 2026 - 18:06:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.2132 iter count: 1.0 +[Apr 27, 2026 - 18:06:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 771939.8125iter count: 1.0 +[Apr 27, 2026 - 18:06:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 14.5948iter count: 1.0 +[Apr 27, 2026 - 18:06:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 13612024.0000iter count: 1.0 +[Apr 27, 2026 - 18:06:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.7012iter count: 1.0 +[Apr 27, 2026 - 18:06:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.7099 iter count: 1.0 +[Apr 27, 2026 - 18:06:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 91101.8125iter count: 1.0 +[Apr 27, 2026 - 18:06:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.7416 iter count: 1.0 +[Apr 27, 2026 - 18:06:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 97074.3594iter count: 1.0 +[Apr 27, 2026 - 18:06:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.8769 iter count: 1.0 +[Apr 27, 2026 - 18:06:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 127239.3438iter count: 1.0 +[Apr 27, 2026 - 18:06:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.2368 iter count: 1.0 +[Apr 27, 2026 - 18:06:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 35367.5781iter count: 1.0 +[Apr 27, 2026 - 18:06:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0191 iter count: 1.0 +[Apr 27, 2026 - 18:06:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 18:06:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.05 seconds +[Apr 27, 2026 - 18:08:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.083045 43.163055 42.987679 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.973145 70.099609 69.917969 +util 98.000000 100.000000 94.000000 +[Apr 27, 2026 - 18:08:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 366 : avg iteration time 82.24 seconds +[Apr 27, 2026 - 18:08:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 366 : data loading time 0.00 +[Apr 27, 2026 - 18:08:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 366 : avg forward pass time 21.64 +[Apr 27, 2026 - 18:08:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 366 : backward pass time 18.14 +[Apr 27, 2026 - 18:08:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 366 : optimizer step time 2.52 +[Apr 27, 2026 - 18:08:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 366-------------------- +[Apr 27, 2026 - 18:08:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0175 iter count: 1.0 +[Apr 27, 2026 - 18:08:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0175 iter count: 1.0 +[Apr 27, 2026 - 18:08:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 18:08:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 18:09:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.082945 43.163055 42.987679 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.973145 70.177734 69.898438 +util 98.000000 100.000000 93.000000 +[Apr 27, 2026 - 18:09:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 367 : avg iteration time 80.34 seconds +[Apr 27, 2026 - 18:09:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 367 : data loading time 0.00 +[Apr 27, 2026 - 18:09:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 367 : avg forward pass time 20.71 +[Apr 27, 2026 - 18:09:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 367 : backward pass time 18.44 +[Apr 27, 2026 - 18:09:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 367 : optimizer step time 2.24 +[Apr 27, 2026 - 18:09:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 367-------------------- +[Apr 27, 2026 - 18:09:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0174 iter count: 1.0 +[Apr 27, 2026 - 18:09:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0174 iter count: 1.0 +[Apr 27, 2026 - 18:09:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 18:09:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.05 seconds +[Apr 27, 2026 - 18:11:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.082952 43.163055 42.987686 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.017090 70.158203 69.917969 +util 96.500000 100.000000 92.000000 +[Apr 27, 2026 - 18:11:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 368 : avg iteration time 86.90 seconds +[Apr 27, 2026 - 18:11:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 368 : data loading time 0.00 +[Apr 27, 2026 - 18:11:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 368 : avg forward pass time 23.32 +[Apr 27, 2026 - 18:11:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 368 : backward pass time 18.62 +[Apr 27, 2026 - 18:11:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 368 : optimizer step time 2.57 +[Apr 27, 2026 - 18:11:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 368-------------------- +[Apr 27, 2026 - 18:11:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0213 iter count: 1.0 +[Apr 27, 2026 - 18:11:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0213 iter count: 1.0 +[Apr 27, 2026 - 18:11:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 18:11:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 18:12:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.082966 43.163082 42.987736 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.046387 70.119141 69.917969 +util 97.500000 100.000000 92.000000 +[Apr 27, 2026 - 18:12:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 369 : avg iteration time 85.89 seconds +[Apr 27, 2026 - 18:12:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 369 : data loading time 0.00 +[Apr 27, 2026 - 18:12:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 369 : avg forward pass time 23.23 +[Apr 27, 2026 - 18:12:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 369 : backward pass time 18.70 +[Apr 27, 2026 - 18:12:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 369 : optimizer step time 2.25 +[Apr 27, 2026 - 18:12:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 369-------------------- +[Apr 27, 2026 - 18:12:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0222 iter count: 1.0 +[Apr 27, 2026 - 18:12:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0222 iter count: 1.0 +[Apr 27, 2026 - 18:12:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 18:12:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.05 seconds +[Apr 27, 2026 - 18:16:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.083000 43.163059 42.987736 +peak_gpu_mem_gb 129.581032 131.620712 123.567759 +peak_gpu_mem_reserved_gb 133.560059 135.595703 127.488281 +util 71.250000 77.000000 66.000000 +[Apr 27, 2026 - 18:16:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 370 : avg iteration time 214.96 seconds +[Apr 27, 2026 - 18:16:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 370 : data loading time 0.00 +[Apr 27, 2026 - 18:16:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 370 : avg forward pass time 98.86 +[Apr 27, 2026 - 18:16:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 370 : backward pass time 8.39 +[Apr 27, 2026 - 18:16:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 370 : optimizer step time 0.34 +[Apr 27, 2026 - 18:16:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 370-------------------- +[Apr 27, 2026 - 18:16:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 329463.7451 iter count: 1.0 +[Apr 27, 2026 - 18:16:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6092 iter count: 1.0 +[Apr 27, 2026 - 18:16:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 329463.7451iter count: 1.0 +[Apr 27, 2026 - 18:16:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 18:16:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.4534iter count: 1.0 +[Apr 27, 2026 - 18:16:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.4860 iter count: 1.0 +[Apr 27, 2026 - 18:16:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.9062 iter count: 1.0 +[Apr 27, 2026 - 18:16:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 474426.0625iter count: 1.0 +[Apr 27, 2026 - 18:16:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 2.6432 iter count: 1.0 +[Apr 27, 2026 - 18:16:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 7365741.0000iter count: 1.0 +[Apr 27, 2026 - 18:16:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6092iter count: 1.0 +[Apr 27, 2026 - 18:16:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 2.4179 iter count: 1.0 +[Apr 27, 2026 - 18:16:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 125.9398iter count: 1.0 +[Apr 27, 2026 - 18:16:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 6.8184 iter count: 1.0 +[Apr 27, 2026 - 18:16:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 836273.6250iter count: 1.0 +[Apr 27, 2026 - 18:16:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.2364 iter count: 1.0 +[Apr 27, 2026 - 18:16:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 647.3068iter count: 1.0 +[Apr 27, 2026 - 18:16:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.9902 iter count: 1.0 +[Apr 27, 2026 - 18:16:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 1179233.6250iter count: 1.0 +[Apr 27, 2026 - 18:16:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0173 iter count: 1.0 +[Apr 27, 2026 - 18:16:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 18:16:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 18:16:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.082993 43.163059 42.987724 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.909180 69.996094 69.843750 +util 97.000000 100.000000 93.000000 +[Apr 27, 2026 - 18:16:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 371 : avg iteration time 49.73 seconds +[Apr 27, 2026 - 18:16:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 371 : data loading time 0.00 +[Apr 27, 2026 - 18:16:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 371 : avg forward pass time 12.18 +[Apr 27, 2026 - 18:16:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 371 : backward pass time 11.55 +[Apr 27, 2026 - 18:16:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 371 : optimizer step time 0.99 +[Apr 27, 2026 - 18:16:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 371-------------------- +[Apr 27, 2026 - 18:16:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0228 iter count: 1.0 +[Apr 27, 2026 - 18:16:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0228 iter count: 1.0 +[Apr 27, 2026 - 18:16:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 18:16:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 18:17:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.083412 43.163078 42.987724 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.933594 70.015625 69.898438 +util 97.750000 99.000000 94.000000 +[Apr 27, 2026 - 18:17:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 372 : avg iteration time 47.72 seconds +[Apr 27, 2026 - 18:17:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 372 : data loading time 0.00 +[Apr 27, 2026 - 18:17:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 372 : avg forward pass time 11.22 +[Apr 27, 2026 - 18:17:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 372 : backward pass time 11.56 +[Apr 27, 2026 - 18:17:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 372 : optimizer step time 0.98 +[Apr 27, 2026 - 18:17:40 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 372-------------------- +[Apr 27, 2026 - 18:17:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0196 iter count: 1.0 +[Apr 27, 2026 - 18:17:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0196 iter count: 1.0 +[Apr 27, 2026 - 18:17:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 18:17:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 18:18:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.083490 43.163082 42.987740 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.972656 70.039062 69.917969 +util 96.000000 100.000000 92.000000 +[Apr 27, 2026 - 18:18:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 373 : avg iteration time 54.29 seconds +[Apr 27, 2026 - 18:18:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 373 : data loading time 0.00 +[Apr 27, 2026 - 18:18:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 373 : avg forward pass time 14.23 +[Apr 27, 2026 - 18:18:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 373 : backward pass time 13.04 +[Apr 27, 2026 - 18:18:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 373 : optimizer step time 1.24 +[Apr 27, 2026 - 18:18:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 373-------------------- +[Apr 27, 2026 - 18:18:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0265 iter count: 1.0 +[Apr 27, 2026 - 18:18:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0265 iter count: 1.0 +[Apr 27, 2026 - 18:18:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 18:18:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 18:19:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.083475 43.163059 42.987747 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.982422 70.078125 69.917969 +util 96.750000 99.000000 93.000000 +[Apr 27, 2026 - 18:19:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 374 : avg iteration time 50.18 seconds +[Apr 27, 2026 - 18:19:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 374 : data loading time 0.00 +[Apr 27, 2026 - 18:19:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 374 : avg forward pass time 13.00 +[Apr 27, 2026 - 18:19:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 374 : backward pass time 11.65 +[Apr 27, 2026 - 18:19:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 374 : optimizer step time 0.97 +[Apr 27, 2026 - 18:19:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 374-------------------- +[Apr 27, 2026 - 18:19:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0265 iter count: 1.0 +[Apr 27, 2026 - 18:19:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0265 iter count: 1.0 +[Apr 27, 2026 - 18:19:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 18:19:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 18:22:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.083473 43.163067 42.987740 +peak_gpu_mem_gb 129.581032 131.620712 123.567759 +peak_gpu_mem_reserved_gb 133.563965 135.619141 127.488281 +util 70.250000 76.000000 65.000000 +[Apr 27, 2026 - 18:22:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 375 : avg iteration time 175.26 seconds +[Apr 27, 2026 - 18:22:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 375 : data loading time 0.00 +[Apr 27, 2026 - 18:22:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 375 : avg forward pass time 79.12 +[Apr 27, 2026 - 18:22:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 375 : backward pass time 8.37 +[Apr 27, 2026 - 18:22:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 375 : optimizer step time 0.34 +[Apr 27, 2026 - 18:22:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 375-------------------- +[Apr 27, 2026 - 18:22:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 1174054.3049 iter count: 1.0 +[Apr 27, 2026 - 18:22:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5110 iter count: 1.0 +[Apr 27, 2026 - 18:22:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 1174054.3049iter count: 1.0 +[Apr 27, 2026 - 18:22:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 18:22:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.3999iter count: 1.0 +[Apr 27, 2026 - 18:22:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.4492 iter count: 1.0 +[Apr 27, 2026 - 18:22:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.9797 iter count: 1.0 +[Apr 27, 2026 - 18:22:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 3032123.0000iter count: 1.0 +[Apr 27, 2026 - 18:22:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 18.1447iter count: 1.0 +[Apr 27, 2026 - 18:22:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 63053348.0000iter count: 1.0 +[Apr 27, 2026 - 18:22:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5110iter count: 1.0 +[Apr 27, 2026 - 18:22:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.7111 iter count: 1.0 +[Apr 27, 2026 - 18:22:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 1672.8484iter count: 1.0 +[Apr 27, 2026 - 18:22:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.3100 iter count: 1.0 +[Apr 27, 2026 - 18:22:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 40942.9141iter count: 1.0 +[Apr 27, 2026 - 18:22:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.7096 iter count: 1.0 +[Apr 27, 2026 - 18:22:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 1667.6781iter count: 1.0 +[Apr 27, 2026 - 18:22:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.5243 iter count: 1.0 +[Apr 27, 2026 - 18:22:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 62851.7734iter count: 1.0 +[Apr 27, 2026 - 18:22:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0197 iter count: 1.0 +[Apr 27, 2026 - 18:22:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 18:22:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 18:23:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.083468 43.163067 42.987724 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.001953 70.152344 69.765625 +util 96.250000 99.000000 92.000000 +[Apr 27, 2026 - 18:23:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 376 : avg iteration time 51.73 seconds +[Apr 27, 2026 - 18:23:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 376 : data loading time 0.00 +[Apr 27, 2026 - 18:23:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 376 : avg forward pass time 13.18 +[Apr 27, 2026 - 18:23:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 376 : backward pass time 11.55 +[Apr 27, 2026 - 18:23:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 376 : optimizer step time 0.98 +[Apr 27, 2026 - 18:23:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 376-------------------- +[Apr 27, 2026 - 18:23:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0184 iter count: 1.0 +[Apr 27, 2026 - 18:23:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0184 iter count: 1.0 +[Apr 27, 2026 - 18:23:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 18:23:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 18:24:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.083473 43.163090 42.987724 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.860352 69.957031 69.609375 +util 97.500000 100.000000 92.000000 +[Apr 27, 2026 - 18:24:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 377 : avg iteration time 51.11 seconds +[Apr 27, 2026 - 18:24:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 377 : data loading time 0.00 +[Apr 27, 2026 - 18:24:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 377 : avg forward pass time 12.82 +[Apr 27, 2026 - 18:24:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 377 : backward pass time 12.31 +[Apr 27, 2026 - 18:24:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 377 : optimizer step time 1.58 +[Apr 27, 2026 - 18:24:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 377-------------------- +[Apr 27, 2026 - 18:24:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0229 iter count: 1.0 +[Apr 27, 2026 - 18:24:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0229 iter count: 1.0 +[Apr 27, 2026 - 18:24:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 18:24:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 18:24:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.083473 43.163090 42.987724 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.909180 69.996094 69.746094 +util 97.000000 100.000000 92.000000 +[Apr 27, 2026 - 18:24:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 378 : avg iteration time 50.26 seconds +[Apr 27, 2026 - 18:24:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 378 : data loading time 0.00 +[Apr 27, 2026 - 18:24:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 378 : avg forward pass time 12.64 +[Apr 27, 2026 - 18:24:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 378 : backward pass time 12.40 +[Apr 27, 2026 - 18:24:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 378 : optimizer step time 1.00 +[Apr 27, 2026 - 18:24:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 378-------------------- +[Apr 27, 2026 - 18:24:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0206 iter count: 1.0 +[Apr 27, 2026 - 18:24:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0206 iter count: 1.0 +[Apr 27, 2026 - 18:24:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 18:24:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 18:25:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.083652 43.163078 42.987724 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.904297 70.054688 69.609375 +util 96.750000 100.000000 92.000000 +[Apr 27, 2026 - 18:25:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 379 : avg iteration time 49.08 seconds +[Apr 27, 2026 - 18:25:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 379 : data loading time 0.00 +[Apr 27, 2026 - 18:25:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 379 : avg forward pass time 12.50 +[Apr 27, 2026 - 18:25:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 379 : backward pass time 11.55 +[Apr 27, 2026 - 18:25:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 379 : optimizer step time 0.98 +[Apr 27, 2026 - 18:25:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 379-------------------- +[Apr 27, 2026 - 18:25:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0211 iter count: 1.0 +[Apr 27, 2026 - 18:25:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0211 iter count: 1.0 +[Apr 27, 2026 - 18:25:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 18:25:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 18:28:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.083858 43.163284 42.987915 +peak_gpu_mem_gb 129.581032 131.620712 123.567759 +peak_gpu_mem_reserved_gb 133.565430 135.599609 127.470703 +util 75.000000 95.000000 63.000000 +[Apr 27, 2026 - 18:28:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 380 : avg iteration time 175.47 seconds +[Apr 27, 2026 - 18:28:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 380 : data loading time 0.00 +[Apr 27, 2026 - 18:28:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 380 : avg forward pass time 78.98 +[Apr 27, 2026 - 18:28:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 380 : backward pass time 8.89 +[Apr 27, 2026 - 18:28:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 380 : optimizer step time 0.35 +[Apr 27, 2026 - 18:28:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 380-------------------- +[Apr 27, 2026 - 18:28:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 104333.7510 iter count: 1.0 +[Apr 27, 2026 - 18:28:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5682 iter count: 1.0 +[Apr 27, 2026 - 18:28:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 104333.7510iter count: 1.0 +[Apr 27, 2026 - 18:28:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 18:28:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.6026iter count: 1.0 +[Apr 27, 2026 - 18:28:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.7819 iter count: 1.0 +[Apr 27, 2026 - 18:28:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.4051 iter count: 1.0 +[Apr 27, 2026 - 18:28:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 231962.1875iter count: 1.0 +[Apr 27, 2026 - 18:28:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 35.2994iter count: 1.0 +[Apr 27, 2026 - 18:28:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 2703737.0000iter count: 1.0 +[Apr 27, 2026 - 18:28:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5682iter count: 1.0 +[Apr 27, 2026 - 18:28:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.8804 iter count: 1.0 +[Apr 27, 2026 - 18:28:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 17341.6465iter count: 1.0 +[Apr 27, 2026 - 18:28:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 6.0497 iter count: 1.0 +[Apr 27, 2026 - 18:28:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 179757.7188iter count: 1.0 +[Apr 27, 2026 - 18:28:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.9555 iter count: 1.0 +[Apr 27, 2026 - 18:28:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 2727.3328iter count: 1.0 +[Apr 27, 2026 - 18:28:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.1238 iter count: 1.0 +[Apr 27, 2026 - 18:28:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 516.8118iter count: 1.0 +[Apr 27, 2026 - 18:28:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0182 iter count: 1.0 +[Apr 27, 2026 - 18:28:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 18:28:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 18:29:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.083863 43.163307 42.987915 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.011719 70.035156 69.980469 +util 96.500000 100.000000 91.000000 +[Apr 27, 2026 - 18:29:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 381 : avg iteration time 51.84 seconds +[Apr 27, 2026 - 18:29:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 381 : data loading time 0.00 +[Apr 27, 2026 - 18:29:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 381 : avg forward pass time 13.81 +[Apr 27, 2026 - 18:29:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 381 : backward pass time 11.68 +[Apr 27, 2026 - 18:29:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 381 : optimizer step time 0.97 +[Apr 27, 2026 - 18:29:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 381-------------------- +[Apr 27, 2026 - 18:29:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0187 iter count: 1.0 +[Apr 27, 2026 - 18:29:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0187 iter count: 1.0 +[Apr 27, 2026 - 18:29:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 18:29:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 18:30:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.083858 43.163284 42.987915 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.962891 69.996094 69.902344 +util 97.500000 99.000000 93.000000 +[Apr 27, 2026 - 18:30:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 382 : avg iteration time 52.78 seconds +[Apr 27, 2026 - 18:30:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 382 : data loading time 0.00 +[Apr 27, 2026 - 18:30:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 382 : avg forward pass time 14.08 +[Apr 27, 2026 - 18:30:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 382 : backward pass time 11.54 +[Apr 27, 2026 - 18:30:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 382 : optimizer step time 0.98 +[Apr 27, 2026 - 18:30:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 382-------------------- +[Apr 27, 2026 - 18:30:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0159 iter count: 1.0 +[Apr 27, 2026 - 18:30:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0159 iter count: 1.0 +[Apr 27, 2026 - 18:30:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 18:30:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 18:31:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.083858 43.163284 42.987915 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.001953 70.058594 69.957031 +util 98.250000 99.000000 96.000000 +[Apr 27, 2026 - 18:31:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 383 : avg iteration time 47.55 seconds +[Apr 27, 2026 - 18:31:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 383 : data loading time 0.00 +[Apr 27, 2026 - 18:31:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 383 : avg forward pass time 11.32 +[Apr 27, 2026 - 18:31:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 383 : backward pass time 11.55 +[Apr 27, 2026 - 18:31:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 383 : optimizer step time 1.02 +[Apr 27, 2026 - 18:31:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 383-------------------- +[Apr 27, 2026 - 18:31:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0147 iter count: 1.0 +[Apr 27, 2026 - 18:31:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0147 iter count: 1.0 +[Apr 27, 2026 - 18:31:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 18:31:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 18:32:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.083868 43.163284 42.987934 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.992188 70.015625 69.957031 +util 97.750000 99.000000 94.000000 +[Apr 27, 2026 - 18:32:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 384 : avg iteration time 53.65 seconds +[Apr 27, 2026 - 18:32:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 384 : data loading time 0.00 +[Apr 27, 2026 - 18:32:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 384 : avg forward pass time 14.17 +[Apr 27, 2026 - 18:32:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 384 : backward pass time 12.71 +[Apr 27, 2026 - 18:32:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 384 : optimizer step time 1.02 +[Apr 27, 2026 - 18:32:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 384-------------------- +[Apr 27, 2026 - 18:32:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0170 iter count: 1.0 +[Apr 27, 2026 - 18:32:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0170 iter count: 1.0 +[Apr 27, 2026 - 18:32:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 18:32:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 18:34:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.084289 43.163284 42.987915 +peak_gpu_mem_gb 129.581032 131.620712 123.567759 +peak_gpu_mem_reserved_gb 133.624023 135.677734 127.548828 +util 66.500000 72.000000 61.000000 +[Apr 27, 2026 - 18:34:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 385 : avg iteration time 175.50 seconds +[Apr 27, 2026 - 18:34:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 385 : data loading time 0.00 +[Apr 27, 2026 - 18:34:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 385 : avg forward pass time 79.24 +[Apr 27, 2026 - 18:34:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 385 : backward pass time 8.39 +[Apr 27, 2026 - 18:34:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 385 : optimizer step time 0.34 +[Apr 27, 2026 - 18:34:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 385-------------------- +[Apr 27, 2026 - 18:34:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 174002.3779 iter count: 1.0 +[Apr 27, 2026 - 18:34:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6758 iter count: 1.0 +[Apr 27, 2026 - 18:34:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 174002.3779iter count: 1.0 +[Apr 27, 2026 - 18:34:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 18:34:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.7439iter count: 1.0 +[Apr 27, 2026 - 18:34:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 2.4418 iter count: 1.0 +[Apr 27, 2026 - 18:34:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.5565 iter count: 1.0 +[Apr 27, 2026 - 18:34:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 268253.5625iter count: 1.0 +[Apr 27, 2026 - 18:34:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 132.1042iter count: 1.0 +[Apr 27, 2026 - 18:34:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 3659897.0000iter count: 1.0 +[Apr 27, 2026 - 18:34:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6758iter count: 1.0 +[Apr 27, 2026 - 18:34:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.1875 iter count: 1.0 +[Apr 27, 2026 - 18:34:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 587.0269iter count: 1.0 +[Apr 27, 2026 - 18:34:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.4555 iter count: 1.0 +[Apr 27, 2026 - 18:34:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 1003.1919iter count: 1.0 +[Apr 27, 2026 - 18:34:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.6693 iter count: 1.0 +[Apr 27, 2026 - 18:34:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 84005.4062iter count: 1.0 +[Apr 27, 2026 - 18:34:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.6409 iter count: 1.0 +[Apr 27, 2026 - 18:34:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 10739.7373iter count: 1.0 +[Apr 27, 2026 - 18:34:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0168 iter count: 1.0 +[Apr 27, 2026 - 18:34:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 18:34:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 18:35:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.084294 43.163307 42.987915 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.559082 72.384766 69.898438 +util 97.500000 99.000000 95.000000 +[Apr 27, 2026 - 18:35:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 386 : avg iteration time 54.54 seconds +[Apr 27, 2026 - 18:35:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 386 : data loading time 0.00 +[Apr 27, 2026 - 18:35:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 386 : avg forward pass time 14.55 +[Apr 27, 2026 - 18:35:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 386 : backward pass time 11.57 +[Apr 27, 2026 - 18:35:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 386 : optimizer step time 0.98 +[Apr 27, 2026 - 18:35:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 386-------------------- +[Apr 27, 2026 - 18:35:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0206 iter count: 1.0 +[Apr 27, 2026 - 18:35:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0206 iter count: 1.0 +[Apr 27, 2026 - 18:35:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 18:35:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 18:36:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.084293 43.163284 42.987915 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.627441 72.560547 69.957031 +util 96.000000 99.000000 92.000000 +[Apr 27, 2026 - 18:36:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 387 : avg iteration time 51.41 seconds +[Apr 27, 2026 - 18:36:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 387 : data loading time 0.00 +[Apr 27, 2026 - 18:36:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 387 : avg forward pass time 12.88 +[Apr 27, 2026 - 18:36:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 387 : backward pass time 12.93 +[Apr 27, 2026 - 18:36:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 387 : optimizer step time 1.17 +[Apr 27, 2026 - 18:36:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 387-------------------- +[Apr 27, 2026 - 18:36:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0166 iter count: 1.0 +[Apr 27, 2026 - 18:36:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0166 iter count: 1.0 +[Apr 27, 2026 - 18:36:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 18:36:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 18:37:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.084348 43.163284 42.987923 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.607910 72.599609 69.937500 +util 96.250000 99.000000 90.000000 +[Apr 27, 2026 - 18:37:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 388 : avg iteration time 49.91 seconds +[Apr 27, 2026 - 18:37:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 388 : data loading time 0.00 +[Apr 27, 2026 - 18:37:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 388 : avg forward pass time 12.88 +[Apr 27, 2026 - 18:37:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 388 : backward pass time 11.55 +[Apr 27, 2026 - 18:37:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 388 : optimizer step time 1.01 +[Apr 27, 2026 - 18:37:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 388-------------------- +[Apr 27, 2026 - 18:37:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0178 iter count: 1.0 +[Apr 27, 2026 - 18:37:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0178 iter count: 1.0 +[Apr 27, 2026 - 18:37:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 18:37:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 18:38:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.084414 43.163284 42.987949 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.642090 72.580078 69.957031 +util 96.000000 99.000000 92.000000 +[Apr 27, 2026 - 18:38:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 389 : avg iteration time 53.35 seconds +[Apr 27, 2026 - 18:38:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 389 : data loading time 0.00 +[Apr 27, 2026 - 18:38:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 389 : avg forward pass time 14.03 +[Apr 27, 2026 - 18:38:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 389 : backward pass time 11.55 +[Apr 27, 2026 - 18:38:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 389 : optimizer step time 0.99 +[Apr 27, 2026 - 18:38:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 389-------------------- +[Apr 27, 2026 - 18:38:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0172 iter count: 1.0 +[Apr 27, 2026 - 18:38:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0172 iter count: 1.0 +[Apr 27, 2026 - 18:38:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 18:38:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 18:41:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.084259 43.163303 42.987934 +peak_gpu_mem_gb 129.581032 131.620712 123.567759 +peak_gpu_mem_reserved_gb 133.608887 135.638672 127.527344 +util 68.000000 73.000000 61.000000 +[Apr 27, 2026 - 18:41:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 390 : avg iteration time 181.46 seconds +[Apr 27, 2026 - 18:41:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 390 : data loading time 0.00 +[Apr 27, 2026 - 18:41:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 390 : avg forward pass time 81.76 +[Apr 27, 2026 - 18:41:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 390 : backward pass time 8.35 +[Apr 27, 2026 - 18:41:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 390 : optimizer step time 0.34 +[Apr 27, 2026 - 18:41:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 390-------------------- +[Apr 27, 2026 - 18:41:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 1664553.1562 iter count: 1.0 +[Apr 27, 2026 - 18:41:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6445 iter count: 1.0 +[Apr 27, 2026 - 18:41:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 1664553.1562iter count: 1.0 +[Apr 27, 2026 - 18:41:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 18:41:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.7285iter count: 1.0 +[Apr 27, 2026 - 18:41:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.4717 iter count: 1.0 +[Apr 27, 2026 - 18:41:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.7569 iter count: 1.0 +[Apr 27, 2026 - 18:41:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 2459271.7500iter count: 1.0 +[Apr 27, 2026 - 18:41:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 2.5685 iter count: 1.0 +[Apr 27, 2026 - 18:41:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 40375776.0000iter count: 1.0 +[Apr 27, 2026 - 18:41:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6445iter count: 1.0 +[Apr 27, 2026 - 18:41:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.6447 iter count: 1.0 +[Apr 27, 2026 - 18:41:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 10822.2764iter count: 1.0 +[Apr 27, 2026 - 18:41:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 6.2601 iter count: 1.0 +[Apr 27, 2026 - 18:41:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 273791.0938iter count: 1.0 +[Apr 27, 2026 - 18:41:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.1600 iter count: 1.0 +[Apr 27, 2026 - 18:41:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 555.5283iter count: 1.0 +[Apr 27, 2026 - 18:41:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.9013 iter count: 1.0 +[Apr 27, 2026 - 18:41:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 18080.8477iter count: 1.0 +[Apr 27, 2026 - 18:41:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0185 iter count: 1.0 +[Apr 27, 2026 - 18:41:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 18:41:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 18:42:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.084253 43.163284 42.987930 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.978027 70.021484 69.957031 +util 95.750000 99.000000 92.000000 +[Apr 27, 2026 - 18:42:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 391 : avg iteration time 51.13 seconds +[Apr 27, 2026 - 18:42:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 391 : data loading time 0.00 +[Apr 27, 2026 - 18:42:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 391 : avg forward pass time 13.38 +[Apr 27, 2026 - 18:42:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 391 : backward pass time 11.54 +[Apr 27, 2026 - 18:42:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 391 : optimizer step time 0.98 +[Apr 27, 2026 - 18:42:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 391-------------------- +[Apr 27, 2026 - 18:42:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0205 iter count: 1.0 +[Apr 27, 2026 - 18:42:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0205 iter count: 1.0 +[Apr 27, 2026 - 18:42:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 18:42:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 18:43:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.084488 43.163296 42.987915 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.987305 70.097656 69.937500 +util 98.250000 100.000000 96.000000 +[Apr 27, 2026 - 18:43:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 392 : avg iteration time 50.84 seconds +[Apr 27, 2026 - 18:43:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 392 : data loading time 0.00 +[Apr 27, 2026 - 18:43:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 392 : avg forward pass time 12.86 +[Apr 27, 2026 - 18:43:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 392 : backward pass time 11.98 +[Apr 27, 2026 - 18:43:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 392 : optimizer step time 1.55 +[Apr 27, 2026 - 18:43:11 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 392-------------------- +[Apr 27, 2026 - 18:43:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0163 iter count: 1.0 +[Apr 27, 2026 - 18:43:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0163 iter count: 1.0 +[Apr 27, 2026 - 18:43:11 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 18:43:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 18:44:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.084494 43.163284 42.987949 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.982422 70.097656 69.917969 +util 97.750000 99.000000 94.000000 +[Apr 27, 2026 - 18:44:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 393 : avg iteration time 50.26 seconds +[Apr 27, 2026 - 18:44:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 393 : data loading time 0.00 +[Apr 27, 2026 - 18:44:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 393 : avg forward pass time 12.66 +[Apr 27, 2026 - 18:44:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 393 : backward pass time 12.38 +[Apr 27, 2026 - 18:44:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 393 : optimizer step time 1.00 +[Apr 27, 2026 - 18:44:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 393-------------------- +[Apr 27, 2026 - 18:44:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0161 iter count: 1.0 +[Apr 27, 2026 - 18:44:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0161 iter count: 1.0 +[Apr 27, 2026 - 18:44:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 18:44:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 18:44:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.084495 43.163284 42.987930 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.958008 70.097656 69.898438 +util 96.000000 99.000000 91.000000 +[Apr 27, 2026 - 18:44:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 394 : avg iteration time 46.26 seconds +[Apr 27, 2026 - 18:44:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 394 : data loading time 0.00 +[Apr 27, 2026 - 18:44:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 394 : avg forward pass time 11.09 +[Apr 27, 2026 - 18:44:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 394 : backward pass time 11.54 +[Apr 27, 2026 - 18:44:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 394 : optimizer step time 0.99 +[Apr 27, 2026 - 18:44:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 394-------------------- +[Apr 27, 2026 - 18:44:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0170 iter count: 1.0 +[Apr 27, 2026 - 18:44:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0170 iter count: 1.0 +[Apr 27, 2026 - 18:44:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 18:44:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 18:47:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.083992 43.163300 42.987915 +peak_gpu_mem_gb 129.581032 131.620712 123.567759 +peak_gpu_mem_reserved_gb 133.593750 135.638672 127.470703 +util 68.750000 74.000000 64.000000 +[Apr 27, 2026 - 18:47:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 395 : avg iteration time 169.66 seconds +[Apr 27, 2026 - 18:47:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 395 : data loading time 0.00 +[Apr 27, 2026 - 18:47:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 395 : avg forward pass time 76.31 +[Apr 27, 2026 - 18:47:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 395 : backward pass time 8.39 +[Apr 27, 2026 - 18:47:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 395 : optimizer step time 0.34 +[Apr 27, 2026 - 18:47:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 395-------------------- +[Apr 27, 2026 - 18:47:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 1552368.9551 iter count: 1.0 +[Apr 27, 2026 - 18:47:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6285 iter count: 1.0 +[Apr 27, 2026 - 18:47:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 1552368.9551iter count: 1.0 +[Apr 27, 2026 - 18:47:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 18:47:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 5.1452iter count: 1.0 +[Apr 27, 2026 - 18:47:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.8064 iter count: 1.0 +[Apr 27, 2026 - 18:47:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 9.0035 iter count: 1.0 +[Apr 27, 2026 - 18:47:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 2906573.0000iter count: 1.0 +[Apr 27, 2026 - 18:47:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 37.0697iter count: 1.0 +[Apr 27, 2026 - 18:47:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 66119688.0000iter count: 1.0 +[Apr 27, 2026 - 18:47:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6285iter count: 1.0 +[Apr 27, 2026 - 18:47:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.5420 iter count: 1.0 +[Apr 27, 2026 - 18:47:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 1192.6636iter count: 1.0 +[Apr 27, 2026 - 18:47:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.4256 iter count: 1.0 +[Apr 27, 2026 - 18:47:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 51595.7422iter count: 1.0 +[Apr 27, 2026 - 18:47:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 6.5519 iter count: 1.0 +[Apr 27, 2026 - 18:47:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 490834.2500iter count: 1.0 +[Apr 27, 2026 - 18:47:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.6314 iter count: 1.0 +[Apr 27, 2026 - 18:47:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 575335.0000iter count: 1.0 +[Apr 27, 2026 - 18:47:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0152 iter count: 1.0 +[Apr 27, 2026 - 18:47:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 18:47:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 18:48:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.083989 43.163284 42.987915 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.012207 70.080078 69.937500 +util 96.500000 100.000000 91.000000 +[Apr 27, 2026 - 18:48:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 396 : avg iteration time 49.83 seconds +[Apr 27, 2026 - 18:48:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 396 : data loading time 0.00 +[Apr 27, 2026 - 18:48:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 396 : avg forward pass time 12.30 +[Apr 27, 2026 - 18:48:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 396 : backward pass time 11.54 +[Apr 27, 2026 - 18:48:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 396 : optimizer step time 0.99 +[Apr 27, 2026 - 18:48:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 396-------------------- +[Apr 27, 2026 - 18:48:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0184 iter count: 1.0 +[Apr 27, 2026 - 18:48:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0184 iter count: 1.0 +[Apr 27, 2026 - 18:48:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 18:48:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 18:49:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.084289 43.163288 42.987926 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.973145 70.021484 69.878906 +util 97.000000 100.000000 92.000000 +[Apr 27, 2026 - 18:49:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 397 : avg iteration time 50.69 seconds +[Apr 27, 2026 - 18:49:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 397 : data loading time 0.00 +[Apr 27, 2026 - 18:49:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 397 : avg forward pass time 12.68 +[Apr 27, 2026 - 18:49:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 397 : backward pass time 11.53 +[Apr 27, 2026 - 18:49:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 397 : optimizer step time 0.99 +[Apr 27, 2026 - 18:49:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 397-------------------- +[Apr 27, 2026 - 18:49:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0168 iter count: 1.0 +[Apr 27, 2026 - 18:49:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0168 iter count: 1.0 +[Apr 27, 2026 - 18:49:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 18:49:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 18:50:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.084287 43.163288 42.987915 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.958496 70.021484 69.898438 +util 97.500000 100.000000 92.000000 +[Apr 27, 2026 - 18:50:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 398 : avg iteration time 54.80 seconds +[Apr 27, 2026 - 18:50:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 398 : data loading time 0.00 +[Apr 27, 2026 - 18:50:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 398 : avg forward pass time 14.36 +[Apr 27, 2026 - 18:50:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 398 : backward pass time 12.94 +[Apr 27, 2026 - 18:50:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 398 : optimizer step time 1.58 +[Apr 27, 2026 - 18:50:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 398-------------------- +[Apr 27, 2026 - 18:50:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0211 iter count: 1.0 +[Apr 27, 2026 - 18:50:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0211 iter count: 1.0 +[Apr 27, 2026 - 18:50:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 18:50:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 18:51:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.084294 43.163292 42.987915 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.065918 70.171875 69.898438 +util 97.000000 99.000000 93.000000 +[Apr 27, 2026 - 18:51:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 399 : avg iteration time 50.80 seconds +[Apr 27, 2026 - 18:51:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 399 : data loading time 0.00 +[Apr 27, 2026 - 18:51:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 399 : avg forward pass time 12.84 +[Apr 27, 2026 - 18:51:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 399 : backward pass time 12.55 +[Apr 27, 2026 - 18:51:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 399 : optimizer step time 1.02 +[Apr 27, 2026 - 18:51:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 399-------------------- +[Apr 27, 2026 - 18:51:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0174 iter count: 1.0 +[Apr 27, 2026 - 18:51:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0174 iter count: 1.0 +[Apr 27, 2026 - 18:51:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 18:51:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 18:53:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.084074 43.163288 42.987915 +peak_gpu_mem_gb 129.581032 131.620712 123.567759 +peak_gpu_mem_reserved_gb 133.614258 135.658203 127.529297 +util 70.000000 75.000000 63.000000 +[Apr 27, 2026 - 18:53:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 400 : avg iteration time 168.74 seconds +[Apr 27, 2026 - 18:53:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 400 : data loading time 0.00 +[Apr 27, 2026 - 18:53:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 400 : avg forward pass time 75.84 +[Apr 27, 2026 - 18:53:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 400 : backward pass time 8.35 +[Apr 27, 2026 - 18:53:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 400 : optimizer step time 0.34 +[Apr 27, 2026 - 18:53:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 400-------------------- +[Apr 27, 2026 - 18:53:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 289346.3555 iter count: 1.0 +[Apr 27, 2026 - 18:53:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6824 iter count: 1.0 +[Apr 27, 2026 - 18:53:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 289346.3555iter count: 1.0 +[Apr 27, 2026 - 18:53:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 18:53:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 5.1129iter count: 1.0 +[Apr 27, 2026 - 18:53:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 2.2645 iter count: 1.0 +[Apr 27, 2026 - 18:53:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.4586 iter count: 1.0 +[Apr 27, 2026 - 18:53:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 431157.4062iter count: 1.0 +[Apr 27, 2026 - 18:53:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 92.6603iter count: 1.0 +[Apr 27, 2026 - 18:53:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 3009039.5000iter count: 1.0 +[Apr 27, 2026 - 18:53:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6824iter count: 1.0 +[Apr 27, 2026 - 18:53:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.3358 iter count: 1.0 +[Apr 27, 2026 - 18:53:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 5835.3530iter count: 1.0 +[Apr 27, 2026 - 18:53:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.9522 iter count: 1.0 +[Apr 27, 2026 - 18:53:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 20020.0117iter count: 1.0 +[Apr 27, 2026 - 18:53:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.7064 iter count: 1.0 +[Apr 27, 2026 - 18:53:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 90468.3203iter count: 1.0 +[Apr 27, 2026 - 18:53:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.6478 iter count: 1.0 +[Apr 27, 2026 - 18:53:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 594577.0625iter count: 1.0 +[Apr 27, 2026 - 18:53:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0170 iter count: 1.0 +[Apr 27, 2026 - 18:53:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[Apr 27, 2026 - 18:54:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 14.98 seconds +[Apr 27, 2026 - 18:54:07 | INFO | fastgen.trainer:save_checkpoint:266 ] Saving checkpoint iteration 400 +[Apr 27, 2026 - 18:54:07 | INFO | fastgen.utils.checkpointer:save:301 ] Saving FSDP model to prefix /tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000400 +[Apr 27, 2026 - 18:54:22 | SUCCESS | fastgen.utils.checkpointer:save:329 ] Model saved at iteration 400 +[Apr 27, 2026 - 18:54:22 | INFO | fastgen.trainer:validate:358 ] Validation iteration 400 +[Apr 27, 2026 - 18:54:23 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 18:54:25 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 2.2s +[Apr 27, 2026 - 18:54:28 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 0: sync_c=6.651 +[Apr 27, 2026 - 18:54:28 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 18:54:30 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.8s +[Apr 27, 2026 - 18:54:33 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 1: sync_c=5.589 +[Apr 27, 2026 - 18:54:34 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 18:54:35 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.8s +[Apr 27, 2026 - 18:54:38 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 2: sync_c=4.788 +[Apr 27, 2026 - 18:54:39 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 18:54:40 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.8s +[Apr 27, 2026 - 18:54:43 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 3: sync_c=4.785 +[Apr 27, 2026 - 18:54:44 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 18:54:46 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.8s +[Apr 27, 2026 - 18:54:49 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 4: sync_c=5.357 +[Apr 27, 2026 - 18:54:49 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 18:54:51 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.8s +[Apr 27, 2026 - 18:54:54 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 5: sync_c=6.218 +[Apr 27, 2026 - 18:54:54 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 18:54:56 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.8s +[Apr 27, 2026 - 18:54:59 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 6: sync_c=6.443 +[Apr 27, 2026 - 18:54:59 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 18:55:01 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.8s +[Apr 27, 2026 - 18:55:04 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 7: sync_c=5.036 +[Apr 27, 2026 - 18:55:04 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 18:55:06 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.8s +[Apr 27, 2026 - 18:55:09 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 8: sync_c=6.140 +[Apr 27, 2026 - 18:55:09 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 18:55:11 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.8s +[Apr 27, 2026 - 18:55:14 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 9: sync_c=3.715 +[Apr 27, 2026 - 18:55:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging val0 stats at iteration 400-------------------- +[Apr 27, 2026 - 18:55:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0000 iter count: 10.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[Apr 27, 2026 - 18:55:31 | INFO | fastgen.callbacks.wandb:on_validation_end:763 ] Logged 10 val videos at iteration 400 +[Apr 27, 2026 - 18:55:31 | INFO | fastgen.callbacks.wandb:on_validation_end:769 ] [SyncEval] val0 mean sync_c: 5.472 (n=10) +[Apr 27, 2026 - 18:56:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.085291 43.163513 42.987305 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.919922 72.769531 70.289062 +util 96.750000 99.000000 93.000000 +[Apr 27, 2026 - 18:56:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 401 : avg iteration time 148.90 seconds +[Apr 27, 2026 - 18:56:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 401 : data loading time 0.00 +[Apr 27, 2026 - 18:56:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 401 : avg forward pass time 12.25 +[Apr 27, 2026 - 18:56:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 401 : backward pass time 12.83 +[Apr 27, 2026 - 18:56:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 401 : optimizer step time 1.06 +[Apr 27, 2026 - 18:56:21 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 401-------------------- +[Apr 27, 2026 - 18:56:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0203 iter count: 1.0 +[Apr 27, 2026 - 18:56:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0203 iter count: 1.0 +[Apr 27, 2026 - 18:56:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 18:56:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 18:57:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.085291 43.163513 42.987305 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.099609 70.175781 69.957031 +util 96.000000 99.000000 92.000000 +[Apr 27, 2026 - 18:57:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 402 : avg iteration time 47.42 seconds +[Apr 27, 2026 - 18:57:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 402 : data loading time 0.00 +[Apr 27, 2026 - 18:57:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 402 : avg forward pass time 11.61 +[Apr 27, 2026 - 18:57:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 402 : backward pass time 11.65 +[Apr 27, 2026 - 18:57:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 402 : optimizer step time 0.97 +[Apr 27, 2026 - 18:57:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 402-------------------- +[Apr 27, 2026 - 18:57:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0166 iter count: 1.0 +[Apr 27, 2026 - 18:57:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0166 iter count: 1.0 +[Apr 27, 2026 - 18:57:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 18:57:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 18:57:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.085320 43.163513 42.987309 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.319336 70.523438 69.957031 +util 96.250000 99.000000 91.000000 +[Apr 27, 2026 - 18:57:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 403 : avg iteration time 50.32 seconds +[Apr 27, 2026 - 18:57:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 403 : data loading time 0.00 +[Apr 27, 2026 - 18:57:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 403 : avg forward pass time 12.80 +[Apr 27, 2026 - 18:57:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 403 : backward pass time 11.54 +[Apr 27, 2026 - 18:57:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 403 : optimizer step time 1.00 +[Apr 27, 2026 - 18:57:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 403-------------------- +[Apr 27, 2026 - 18:57:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0192 iter count: 1.0 +[Apr 27, 2026 - 18:57:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0192 iter count: 1.0 +[Apr 27, 2026 - 18:57:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 18:57:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 18:58:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.085319 43.163513 42.987305 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.270508 70.464844 69.957031 +util 95.000000 99.000000 91.000000 +[Apr 27, 2026 - 18:58:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 404 : avg iteration time 50.87 seconds +[Apr 27, 2026 - 18:58:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 404 : data loading time 0.00 +[Apr 27, 2026 - 18:58:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 404 : avg forward pass time 12.62 +[Apr 27, 2026 - 18:58:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 404 : backward pass time 11.54 +[Apr 27, 2026 - 18:58:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 404 : optimizer step time 0.98 +[Apr 27, 2026 - 18:58:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 404-------------------- +[Apr 27, 2026 - 18:58:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0189 iter count: 1.0 +[Apr 27, 2026 - 18:58:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0189 iter count: 1.0 +[Apr 27, 2026 - 18:58:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 18:58:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 19:01:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.084579 43.163971 42.987755 +peak_gpu_mem_gb 129.581032 131.620712 123.567759 +peak_gpu_mem_reserved_gb 133.525879 135.521484 127.546875 +util 71.750000 78.000000 65.000000 +[Apr 27, 2026 - 19:01:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 405 : avg iteration time 179.19 seconds +[Apr 27, 2026 - 19:01:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 405 : data loading time 0.00 +[Apr 27, 2026 - 19:01:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 405 : avg forward pass time 80.30 +[Apr 27, 2026 - 19:01:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 405 : backward pass time 9.96 +[Apr 27, 2026 - 19:01:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 405 : optimizer step time 0.34 +[Apr 27, 2026 - 19:01:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 405-------------------- +[Apr 27, 2026 - 19:01:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 428849.1372 iter count: 1.0 +[Apr 27, 2026 - 19:01:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5742 iter count: 1.0 +[Apr 27, 2026 - 19:01:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 428849.1372iter count: 1.0 +[Apr 27, 2026 - 19:01:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:01:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.5259iter count: 1.0 +[Apr 27, 2026 - 19:01:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.8011 iter count: 1.0 +[Apr 27, 2026 - 19:01:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.5769 iter count: 1.0 +[Apr 27, 2026 - 19:01:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 984105.0000iter count: 1.0 +[Apr 27, 2026 - 19:01:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 36.6766iter count: 1.0 +[Apr 27, 2026 - 19:01:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 28168132.0000iter count: 1.0 +[Apr 27, 2026 - 19:01:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5742iter count: 1.0 +[Apr 27, 2026 - 19:01:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 2.3912 iter count: 1.0 +[Apr 27, 2026 - 19:01:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 119.3942iter count: 1.0 +[Apr 27, 2026 - 19:01:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 6.2187 iter count: 1.0 +[Apr 27, 2026 - 19:01:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 252035.0781iter count: 1.0 +[Apr 27, 2026 - 19:01:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.1803 iter count: 1.0 +[Apr 27, 2026 - 19:01:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 4275.1592iter count: 1.0 +[Apr 27, 2026 - 19:01:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.6443 iter count: 1.0 +[Apr 27, 2026 - 19:01:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 10814.4160iter count: 1.0 +[Apr 27, 2026 - 19:01:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0211 iter count: 1.0 +[Apr 27, 2026 - 19:01:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:01:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 19:02:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.084581 43.163963 42.987755 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.879883 70.035156 69.781250 +util 95.750000 99.000000 91.000000 +[Apr 27, 2026 - 19:02:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 406 : avg iteration time 51.98 seconds +[Apr 27, 2026 - 19:02:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 406 : data loading time 0.00 +[Apr 27, 2026 - 19:02:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 406 : avg forward pass time 13.80 +[Apr 27, 2026 - 19:02:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 406 : backward pass time 11.85 +[Apr 27, 2026 - 19:02:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 406 : optimizer step time 0.98 +[Apr 27, 2026 - 19:02:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 406-------------------- +[Apr 27, 2026 - 19:02:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0174 iter count: 1.0 +[Apr 27, 2026 - 19:02:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0174 iter count: 1.0 +[Apr 27, 2026 - 19:02:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:02:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 19:03:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.084931 43.163963 42.987755 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.899414 70.000000 69.800781 +util 96.750000 99.000000 92.000000 +[Apr 27, 2026 - 19:03:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 407 : avg iteration time 49.97 seconds +[Apr 27, 2026 - 19:03:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 407 : data loading time 0.00 +[Apr 27, 2026 - 19:03:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 407 : avg forward pass time 12.72 +[Apr 27, 2026 - 19:03:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 407 : backward pass time 11.57 +[Apr 27, 2026 - 19:03:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 407 : optimizer step time 0.98 +[Apr 27, 2026 - 19:03:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 407-------------------- +[Apr 27, 2026 - 19:03:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0167 iter count: 1.0 +[Apr 27, 2026 - 19:03:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0167 iter count: 1.0 +[Apr 27, 2026 - 19:03:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:03:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 19:04:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.085053 43.163963 42.987755 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.821289 70.039062 69.722656 +util 97.750000 99.000000 94.000000 +[Apr 27, 2026 - 19:04:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 408 : avg iteration time 47.97 seconds +[Apr 27, 2026 - 19:04:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 408 : data loading time 0.00 +[Apr 27, 2026 - 19:04:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 408 : avg forward pass time 11.15 +[Apr 27, 2026 - 19:04:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 408 : backward pass time 11.61 +[Apr 27, 2026 - 19:04:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 408 : optimizer step time 0.99 +[Apr 27, 2026 - 19:04:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 408-------------------- +[Apr 27, 2026 - 19:04:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0173 iter count: 1.0 +[Apr 27, 2026 - 19:04:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0173 iter count: 1.0 +[Apr 27, 2026 - 19:04:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:04:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 19:05:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.085061 43.163963 42.987755 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.899414 70.039062 69.800781 +util 98.750000 100.000000 96.000000 +[Apr 27, 2026 - 19:05:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 409 : avg iteration time 50.55 seconds +[Apr 27, 2026 - 19:05:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 409 : data loading time 0.00 +[Apr 27, 2026 - 19:05:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 409 : avg forward pass time 12.62 +[Apr 27, 2026 - 19:05:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 409 : backward pass time 11.59 +[Apr 27, 2026 - 19:05:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 409 : optimizer step time 1.33 +[Apr 27, 2026 - 19:05:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 409-------------------- +[Apr 27, 2026 - 19:05:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0200 iter count: 1.0 +[Apr 27, 2026 - 19:05:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0200 iter count: 1.0 +[Apr 27, 2026 - 19:05:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:05:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 19:08:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.084722 43.163067 42.987759 +peak_gpu_mem_gb 129.581032 131.620712 123.567759 +peak_gpu_mem_reserved_gb 133.555664 135.697266 127.490234 +util 71.250000 78.000000 63.000000 +[Apr 27, 2026 - 19:08:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 410 : avg iteration time 173.30 seconds +[Apr 27, 2026 - 19:08:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 410 : data loading time 0.00 +[Apr 27, 2026 - 19:08:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 410 : avg forward pass time 78.13 +[Apr 27, 2026 - 19:08:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 410 : backward pass time 8.35 +[Apr 27, 2026 - 19:08:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 410 : optimizer step time 0.34 +[Apr 27, 2026 - 19:08:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 410-------------------- +[Apr 27, 2026 - 19:08:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 155679.5210 iter count: 1.0 +[Apr 27, 2026 - 19:08:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5438 iter count: 1.0 +[Apr 27, 2026 - 19:08:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 155679.5210iter count: 1.0 +[Apr 27, 2026 - 19:08:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:08:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.4123iter count: 1.0 +[Apr 27, 2026 - 19:08:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.9302 iter count: 1.0 +[Apr 27, 2026 - 19:08:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.6892 iter count: 1.0 +[Apr 27, 2026 - 19:08:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 311134.0938iter count: 1.0 +[Apr 27, 2026 - 19:08:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 6.4267 iter count: 1.0 +[Apr 27, 2026 - 19:08:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 4772818.5000iter count: 1.0 +[Apr 27, 2026 - 19:08:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5438iter count: 1.0 +[Apr 27, 2026 - 19:08:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.1917 iter count: 1.0 +[Apr 27, 2026 - 19:08:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 591.8816iter count: 1.0 +[Apr 27, 2026 - 19:08:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.4161 iter count: 1.0 +[Apr 27, 2026 - 19:08:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 927.2418iter count: 1.0 +[Apr 27, 2026 - 19:08:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.3517 iter count: 1.0 +[Apr 27, 2026 - 19:08:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 815.1755iter count: 1.0 +[Apr 27, 2026 - 19:08:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.0533 iter count: 1.0 +[Apr 27, 2026 - 19:08:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 3316.2957iter count: 1.0 +[Apr 27, 2026 - 19:08:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0206 iter count: 1.0 +[Apr 27, 2026 - 19:08:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:08:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 19:08:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.084724 43.163067 42.987770 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.958008 70.039062 69.917969 +util 96.000000 99.000000 90.000000 +[Apr 27, 2026 - 19:08:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 411 : avg iteration time 55.17 seconds +[Apr 27, 2026 - 19:08:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 411 : data loading time 0.00 +[Apr 27, 2026 - 19:08:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 411 : avg forward pass time 14.89 +[Apr 27, 2026 - 19:08:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 411 : backward pass time 11.55 +[Apr 27, 2026 - 19:08:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 411 : optimizer step time 1.00 +[Apr 27, 2026 - 19:08:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 411-------------------- +[Apr 27, 2026 - 19:08:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0175 iter count: 1.0 +[Apr 27, 2026 - 19:08:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0175 iter count: 1.0 +[Apr 27, 2026 - 19:08:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:08:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 19:09:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.084722 43.163071 42.987755 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.958008 70.078125 69.917969 +util 97.500000 100.000000 94.000000 +[Apr 27, 2026 - 19:09:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 412 : avg iteration time 50.70 seconds +[Apr 27, 2026 - 19:09:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 412 : data loading time 0.00 +[Apr 27, 2026 - 19:09:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 412 : avg forward pass time 12.69 +[Apr 27, 2026 - 19:09:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 412 : backward pass time 11.69 +[Apr 27, 2026 - 19:09:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 412 : optimizer step time 1.44 +[Apr 27, 2026 - 19:09:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 412-------------------- +[Apr 27, 2026 - 19:09:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0182 iter count: 1.0 +[Apr 27, 2026 - 19:09:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0182 iter count: 1.0 +[Apr 27, 2026 - 19:09:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:09:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 19:10:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.084721 43.163067 42.987755 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.972656 70.136719 69.917969 +util 95.500000 99.000000 91.000000 +[Apr 27, 2026 - 19:10:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 413 : avg iteration time 50.53 seconds +[Apr 27, 2026 - 19:10:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 413 : data loading time 0.00 +[Apr 27, 2026 - 19:10:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 413 : avg forward pass time 12.67 +[Apr 27, 2026 - 19:10:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 413 : backward pass time 12.61 +[Apr 27, 2026 - 19:10:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 413 : optimizer step time 1.04 +[Apr 27, 2026 - 19:10:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 413-------------------- +[Apr 27, 2026 - 19:10:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0150 iter count: 1.0 +[Apr 27, 2026 - 19:10:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0150 iter count: 1.0 +[Apr 27, 2026 - 19:10:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:10:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 19:11:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.084790 43.163071 42.987759 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.918945 70.156250 69.800781 +util 96.500000 100.000000 92.000000 +[Apr 27, 2026 - 19:11:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 414 : avg iteration time 49.21 seconds +[Apr 27, 2026 - 19:11:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 414 : data loading time 0.00 +[Apr 27, 2026 - 19:11:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 414 : avg forward pass time 12.55 +[Apr 27, 2026 - 19:11:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 414 : backward pass time 11.56 +[Apr 27, 2026 - 19:11:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 414 : optimizer step time 0.99 +[Apr 27, 2026 - 19:11:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 414-------------------- +[Apr 27, 2026 - 19:11:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0151 iter count: 1.0 +[Apr 27, 2026 - 19:11:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0151 iter count: 1.0 +[Apr 27, 2026 - 19:11:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:11:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 19:14:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.084633 43.163082 42.987759 +peak_gpu_mem_gb 129.581032 131.620712 123.567759 +peak_gpu_mem_reserved_gb 133.565430 135.595703 127.548828 +util 71.500000 77.000000 67.000000 +[Apr 27, 2026 - 19:14:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 415 : avg iteration time 173.08 seconds +[Apr 27, 2026 - 19:14:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 415 : data loading time 0.00 +[Apr 27, 2026 - 19:14:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 415 : avg forward pass time 78.02 +[Apr 27, 2026 - 19:14:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 415 : backward pass time 8.39 +[Apr 27, 2026 - 19:14:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 415 : optimizer step time 0.34 +[Apr 27, 2026 - 19:14:21 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 415-------------------- +[Apr 27, 2026 - 19:14:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 794228.8973 iter count: 1.0 +[Apr 27, 2026 - 19:14:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6607 iter count: 1.0 +[Apr 27, 2026 - 19:14:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 794228.8973iter count: 1.0 +[Apr 27, 2026 - 19:14:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:14:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.6473iter count: 1.0 +[Apr 27, 2026 - 19:14:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.5331 iter count: 1.0 +[Apr 27, 2026 - 19:14:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.5448 iter count: 1.0 +[Apr 27, 2026 - 19:14:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 954184.5625iter count: 1.0 +[Apr 27, 2026 - 19:14:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 21.4610iter count: 1.0 +[Apr 27, 2026 - 19:14:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 26418142.0000iter count: 1.0 +[Apr 27, 2026 - 19:14:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6607iter count: 1.0 +[Apr 27, 2026 - 19:14:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.8560 iter count: 1.0 +[Apr 27, 2026 - 19:14:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 2235.0930iter count: 1.0 +[Apr 27, 2026 - 19:14:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.9171 iter count: 1.0 +[Apr 27, 2026 - 19:14:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 18659.8125iter count: 1.0 +[Apr 27, 2026 - 19:14:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.2663 iter count: 1.0 +[Apr 27, 2026 - 19:14:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 5077.2646iter count: 1.0 +[Apr 27, 2026 - 19:14:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 1.5331 iter count: 1.0 +[Apr 27, 2026 - 19:14:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 21.4610 iter count: 1.0 +[Apr 27, 2026 - 19:14:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0150 iter count: 1.0 +[Apr 27, 2026 - 19:14:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:14:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 19:15:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.084935 43.163082 42.987759 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.685547 72.808594 69.898438 +util 96.000000 99.000000 91.000000 +[Apr 27, 2026 - 19:15:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 416 : avg iteration time 51.68 seconds +[Apr 27, 2026 - 19:15:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 416 : data loading time 0.00 +[Apr 27, 2026 - 19:15:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 416 : avg forward pass time 13.26 +[Apr 27, 2026 - 19:15:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 416 : backward pass time 11.53 +[Apr 27, 2026 - 19:15:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 416 : optimizer step time 1.02 +[Apr 27, 2026 - 19:15:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 416-------------------- +[Apr 27, 2026 - 19:15:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0166 iter count: 1.0 +[Apr 27, 2026 - 19:15:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0166 iter count: 1.0 +[Apr 27, 2026 - 19:15:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:15:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 19:16:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.084935 43.163082 42.987759 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.680664 72.828125 69.859375 +util 98.500000 100.000000 96.000000 +[Apr 27, 2026 - 19:16:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 417 : avg iteration time 48.52 seconds +[Apr 27, 2026 - 19:16:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 417 : data loading time 0.00 +[Apr 27, 2026 - 19:16:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 417 : avg forward pass time 11.55 +[Apr 27, 2026 - 19:16:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 417 : backward pass time 12.32 +[Apr 27, 2026 - 19:16:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 417 : optimizer step time 1.58 +[Apr 27, 2026 - 19:16:02 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 417-------------------- +[Apr 27, 2026 - 19:16:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0155 iter count: 1.0 +[Apr 27, 2026 - 19:16:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0155 iter count: 1.0 +[Apr 27, 2026 - 19:16:02 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:16:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 19:16:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.084997 43.163082 42.987759 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.583008 72.769531 69.742188 +util 98.250000 99.000000 96.000000 +[Apr 27, 2026 - 19:16:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 418 : avg iteration time 52.80 seconds +[Apr 27, 2026 - 19:16:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 418 : data loading time 0.00 +[Apr 27, 2026 - 19:16:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 418 : avg forward pass time 14.32 +[Apr 27, 2026 - 19:16:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 418 : backward pass time 11.62 +[Apr 27, 2026 - 19:16:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 418 : optimizer step time 0.98 +[Apr 27, 2026 - 19:16:54 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 418-------------------- +[Apr 27, 2026 - 19:16:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0169 iter count: 1.0 +[Apr 27, 2026 - 19:16:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0169 iter count: 1.0 +[Apr 27, 2026 - 19:16:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:16:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 19:17:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.085001 43.163082 42.987759 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.680664 72.789062 69.878906 +util 97.000000 99.000000 92.000000 +[Apr 27, 2026 - 19:17:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 419 : avg iteration time 53.09 seconds +[Apr 27, 2026 - 19:17:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 419 : data loading time 0.00 +[Apr 27, 2026 - 19:17:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 419 : avg forward pass time 14.14 +[Apr 27, 2026 - 19:17:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 419 : backward pass time 11.58 +[Apr 27, 2026 - 19:17:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 419 : optimizer step time 0.99 +[Apr 27, 2026 - 19:17:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 419-------------------- +[Apr 27, 2026 - 19:17:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0189 iter count: 1.0 +[Apr 27, 2026 - 19:17:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0189 iter count: 1.0 +[Apr 27, 2026 - 19:17:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:17:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 19:20:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.085222 43.163082 42.987774 +peak_gpu_mem_gb 129.581032 131.620712 123.567759 +peak_gpu_mem_reserved_gb 133.722168 135.599609 128.292969 +util 65.500000 71.000000 61.000000 +[Apr 27, 2026 - 19:20:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 420 : avg iteration time 176.85 seconds +[Apr 27, 2026 - 19:20:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 420 : data loading time 0.00 +[Apr 27, 2026 - 19:20:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 420 : avg forward pass time 79.86 +[Apr 27, 2026 - 19:20:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 420 : backward pass time 8.44 +[Apr 27, 2026 - 19:20:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 420 : optimizer step time 0.34 +[Apr 27, 2026 - 19:20:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 420-------------------- +[Apr 27, 2026 - 19:20:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 246228.8203 iter count: 1.0 +[Apr 27, 2026 - 19:20:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6976 iter count: 1.0 +[Apr 27, 2026 - 19:20:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 246228.8203iter count: 1.0 +[Apr 27, 2026 - 19:20:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:20:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 5.1261iter count: 1.0 +[Apr 27, 2026 - 19:20:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 2.2669 iter count: 1.0 +[Apr 27, 2026 - 19:20:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.8036 iter count: 1.0 +[Apr 27, 2026 - 19:20:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 421168.6250iter count: 1.0 +[Apr 27, 2026 - 19:20:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 93.1115iter count: 1.0 +[Apr 27, 2026 - 19:20:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 5999578.0000iter count: 1.0 +[Apr 27, 2026 - 19:20:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6976iter count: 1.0 +[Apr 27, 2026 - 19:20:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 6.3109 iter count: 1.0 +[Apr 27, 2026 - 19:20:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 303068.3438iter count: 1.0 +[Apr 27, 2026 - 19:20:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 6.2438 iter count: 1.0 +[Apr 27, 2026 - 19:20:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 265051.2500iter count: 1.0 +[Apr 27, 2026 - 19:20:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.6210 iter count: 1.0 +[Apr 27, 2026 - 19:20:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 76274.5781iter count: 1.0 +[Apr 27, 2026 - 19:20:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.2453 iter count: 1.0 +[Apr 27, 2026 - 19:20:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 265805.8438iter count: 1.0 +[Apr 27, 2026 - 19:20:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0166 iter count: 1.0 +[Apr 27, 2026 - 19:20:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:20:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 19:21:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.085206 43.163082 42.987759 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.899414 70.000000 69.800781 +util 96.500000 99.000000 93.000000 +[Apr 27, 2026 - 19:21:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 421 : avg iteration time 53.99 seconds +[Apr 27, 2026 - 19:21:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 421 : data loading time 0.00 +[Apr 27, 2026 - 19:21:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 421 : avg forward pass time 14.75 +[Apr 27, 2026 - 19:21:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 421 : backward pass time 11.57 +[Apr 27, 2026 - 19:21:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 421 : optimizer step time 0.98 +[Apr 27, 2026 - 19:21:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 421-------------------- +[Apr 27, 2026 - 19:21:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0172 iter count: 1.0 +[Apr 27, 2026 - 19:21:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0172 iter count: 1.0 +[Apr 27, 2026 - 19:21:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:21:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 19:22:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.085210 43.163082 42.987759 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.948242 70.035156 69.917969 +util 97.250000 99.000000 94.000000 +[Apr 27, 2026 - 19:22:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 422 : avg iteration time 48.25 seconds +[Apr 27, 2026 - 19:22:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 422 : data loading time 0.00 +[Apr 27, 2026 - 19:22:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 422 : avg forward pass time 11.52 +[Apr 27, 2026 - 19:22:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 422 : backward pass time 11.56 +[Apr 27, 2026 - 19:22:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 422 : optimizer step time 0.98 +[Apr 27, 2026 - 19:22:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 422-------------------- +[Apr 27, 2026 - 19:22:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0162 iter count: 1.0 +[Apr 27, 2026 - 19:22:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0162 iter count: 1.0 +[Apr 27, 2026 - 19:22:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:22:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 19:23:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.085267 43.163082 42.987759 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.918945 70.035156 69.859375 +util 97.250000 99.000000 94.000000 +[Apr 27, 2026 - 19:23:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 423 : avg iteration time 50.41 seconds +[Apr 27, 2026 - 19:23:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 423 : data loading time 0.00 +[Apr 27, 2026 - 19:23:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 423 : avg forward pass time 12.52 +[Apr 27, 2026 - 19:23:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 423 : backward pass time 11.54 +[Apr 27, 2026 - 19:23:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 423 : optimizer step time 1.02 +[Apr 27, 2026 - 19:23:17 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 423-------------------- +[Apr 27, 2026 - 19:23:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0188 iter count: 1.0 +[Apr 27, 2026 - 19:23:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0188 iter count: 1.0 +[Apr 27, 2026 - 19:23:17 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:23:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 19:24:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.085267 43.163082 42.987759 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.918945 69.996094 69.878906 +util 97.000000 99.000000 92.000000 +[Apr 27, 2026 - 19:24:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 424 : avg iteration time 50.51 seconds +[Apr 27, 2026 - 19:24:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 424 : data loading time 0.00 +[Apr 27, 2026 - 19:24:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 424 : avg forward pass time 12.58 +[Apr 27, 2026 - 19:24:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 424 : backward pass time 12.76 +[Apr 27, 2026 - 19:24:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 424 : optimizer step time 1.05 +[Apr 27, 2026 - 19:24:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 424-------------------- +[Apr 27, 2026 - 19:24:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0165 iter count: 1.0 +[Apr 27, 2026 - 19:24:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0165 iter count: 1.0 +[Apr 27, 2026 - 19:24:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:24:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 19:26:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.085206 43.163082 42.987759 +peak_gpu_mem_gb 129.581032 131.620712 123.567759 +peak_gpu_mem_reserved_gb 133.888184 135.677734 128.683594 +util 68.750000 77.000000 63.000000 +[Apr 27, 2026 - 19:26:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 425 : avg iteration time 168.83 seconds +[Apr 27, 2026 - 19:26:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 425 : data loading time 0.00 +[Apr 27, 2026 - 19:26:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 425 : avg forward pass time 75.90 +[Apr 27, 2026 - 19:26:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 425 : backward pass time 8.39 +[Apr 27, 2026 - 19:26:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 425 : optimizer step time 0.34 +[Apr 27, 2026 - 19:26:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 425-------------------- +[Apr 27, 2026 - 19:26:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 28965.0305 iter count: 1.0 +[Apr 27, 2026 - 19:26:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5764 iter count: 1.0 +[Apr 27, 2026 - 19:26:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 28965.0305iter count: 1.0 +[Apr 27, 2026 - 19:26:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:26:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 3.9880iter count: 1.0 +[Apr 27, 2026 - 19:26:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.4632 iter count: 1.0 +[Apr 27, 2026 - 19:26:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 6.9177 iter count: 1.0 +[Apr 27, 2026 - 19:26:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 52495.7031iter count: 1.0 +[Apr 27, 2026 - 19:26:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 2.5256 iter count: 1.0 +[Apr 27, 2026 - 19:26:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 1020180.5625iter count: 1.0 +[Apr 27, 2026 - 19:26:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5764iter count: 1.0 +[Apr 27, 2026 - 19:26:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.8250 iter count: 1.0 +[Apr 27, 2026 - 19:26:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 15522.8779iter count: 1.0 +[Apr 27, 2026 - 19:26:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.1195 iter count: 1.0 +[Apr 27, 2026 - 19:26:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 3785.6023iter count: 1.0 +[Apr 27, 2026 - 19:26:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 2.4558 iter count: 1.0 +[Apr 27, 2026 - 19:26:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 135.8657iter count: 1.0 +[Apr 27, 2026 - 19:26:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.9177 iter count: 1.0 +[Apr 27, 2026 - 19:26:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 1020180.5625iter count: 1.0 +[Apr 27, 2026 - 19:26:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0161 iter count: 1.0 +[Apr 27, 2026 - 19:26:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:26:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 19:27:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.085200 43.163082 42.987759 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.982910 70.138672 69.917969 +util 97.250000 100.000000 92.000000 +[Apr 27, 2026 - 19:27:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 426 : avg iteration time 54.37 seconds +[Apr 27, 2026 - 19:27:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 426 : data loading time 0.00 +[Apr 27, 2026 - 19:27:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 426 : avg forward pass time 14.48 +[Apr 27, 2026 - 19:27:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 426 : backward pass time 11.54 +[Apr 27, 2026 - 19:27:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 426 : optimizer step time 0.97 +[Apr 27, 2026 - 19:27:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 426-------------------- +[Apr 27, 2026 - 19:27:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0182 iter count: 1.0 +[Apr 27, 2026 - 19:27:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0182 iter count: 1.0 +[Apr 27, 2026 - 19:27:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:27:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 19:28:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.085646 43.163101 42.987759 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.963379 70.041016 69.917969 +util 98.000000 99.000000 95.000000 +[Apr 27, 2026 - 19:28:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 427 : avg iteration time 53.98 seconds +[Apr 27, 2026 - 19:28:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 427 : data loading time 0.00 +[Apr 27, 2026 - 19:28:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 427 : avg forward pass time 14.23 +[Apr 27, 2026 - 19:28:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 427 : backward pass time 12.85 +[Apr 27, 2026 - 19:28:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 427 : optimizer step time 1.12 +[Apr 27, 2026 - 19:28:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 427-------------------- +[Apr 27, 2026 - 19:28:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0182 iter count: 1.0 +[Apr 27, 2026 - 19:28:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0182 iter count: 1.0 +[Apr 27, 2026 - 19:28:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:28:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 19:29:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.085633 43.163082 42.987759 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.982910 70.041016 69.957031 +util 96.250000 100.000000 92.000000 +[Apr 27, 2026 - 19:29:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 428 : avg iteration time 49.84 seconds +[Apr 27, 2026 - 19:29:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 428 : data loading time 0.00 +[Apr 27, 2026 - 19:29:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 428 : avg forward pass time 12.89 +[Apr 27, 2026 - 19:29:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 428 : backward pass time 11.52 +[Apr 27, 2026 - 19:29:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 428 : optimizer step time 0.99 +[Apr 27, 2026 - 19:29:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 428-------------------- +[Apr 27, 2026 - 19:29:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0170 iter count: 1.0 +[Apr 27, 2026 - 19:29:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0170 iter count: 1.0 +[Apr 27, 2026 - 19:29:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:29:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 19:30:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.085633 43.163082 42.987759 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.982910 70.041016 69.937500 +util 97.250000 99.000000 93.000000 +[Apr 27, 2026 - 19:30:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 429 : avg iteration time 50.39 seconds +[Apr 27, 2026 - 19:30:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 429 : data loading time 0.00 +[Apr 27, 2026 - 19:30:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 429 : avg forward pass time 12.73 +[Apr 27, 2026 - 19:30:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 429 : backward pass time 11.55 +[Apr 27, 2026 - 19:30:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 429 : optimizer step time 0.99 +[Apr 27, 2026 - 19:30:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 429-------------------- +[Apr 27, 2026 - 19:30:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0175 iter count: 1.0 +[Apr 27, 2026 - 19:30:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0175 iter count: 1.0 +[Apr 27, 2026 - 19:30:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:30:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 19:33:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.085433 43.163082 42.987759 +peak_gpu_mem_gb 129.581032 131.620712 123.567759 +peak_gpu_mem_reserved_gb 133.536133 135.560547 127.548828 +util 72.000000 79.000000 64.000000 +[Apr 27, 2026 - 19:33:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 430 : avg iteration time 177.21 seconds +[Apr 27, 2026 - 19:33:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 430 : data loading time 0.00 +[Apr 27, 2026 - 19:33:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 430 : avg forward pass time 79.79 +[Apr 27, 2026 - 19:33:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 430 : backward pass time 8.43 +[Apr 27, 2026 - 19:33:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 430 : optimizer step time 0.34 +[Apr 27, 2026 - 19:33:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 430-------------------- +[Apr 27, 2026 - 19:33:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 814564.5488 iter count: 1.0 +[Apr 27, 2026 - 19:33:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6180 iter count: 1.0 +[Apr 27, 2026 - 19:33:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 814564.5488iter count: 1.0 +[Apr 27, 2026 - 19:33:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:33:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.7129iter count: 1.0 +[Apr 27, 2026 - 19:33:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.4132 iter count: 1.0 +[Apr 27, 2026 - 19:33:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.5127 iter count: 1.0 +[Apr 27, 2026 - 19:33:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 964171.7500iter count: 1.0 +[Apr 27, 2026 - 19:33:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 16.8847iter count: 1.0 +[Apr 27, 2026 - 19:33:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 24778620.0000iter count: 1.0 +[Apr 27, 2026 - 19:33:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6180iter count: 1.0 +[Apr 27, 2026 - 19:33:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.7191 iter count: 1.0 +[Apr 27, 2026 - 19:33:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 1699.7042iter count: 1.0 +[Apr 27, 2026 - 19:33:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.5048 iter count: 1.0 +[Apr 27, 2026 - 19:33:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 1107.1921iter count: 1.0 +[Apr 27, 2026 - 19:33:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 6.3478 iter count: 1.0 +[Apr 27, 2026 - 19:33:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 326297.3125iter count: 1.0 +[Apr 27, 2026 - 19:33:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 2.0100 iter count: 1.0 +[Apr 27, 2026 - 19:33:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 55.6971 iter count: 1.0 +[Apr 27, 2026 - 19:33:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0163 iter count: 1.0 +[Apr 27, 2026 - 19:33:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:33:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 19:34:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.085443 43.163082 42.987759 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.026367 70.175781 69.937500 +util 98.000000 99.000000 95.000000 +[Apr 27, 2026 - 19:34:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 431 : avg iteration time 51.35 seconds +[Apr 27, 2026 - 19:34:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 431 : data loading time 0.00 +[Apr 27, 2026 - 19:34:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 431 : avg forward pass time 13.41 +[Apr 27, 2026 - 19:34:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 431 : backward pass time 11.59 +[Apr 27, 2026 - 19:34:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 431 : optimizer step time 0.98 +[Apr 27, 2026 - 19:34:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 431-------------------- +[Apr 27, 2026 - 19:34:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0166 iter count: 1.0 +[Apr 27, 2026 - 19:34:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0166 iter count: 1.0 +[Apr 27, 2026 - 19:34:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:34:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 19:35:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.085674 43.163082 42.987762 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.001953 70.097656 69.937500 +util 97.250000 99.000000 92.000000 +[Apr 27, 2026 - 19:35:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 432 : avg iteration time 50.18 seconds +[Apr 27, 2026 - 19:35:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 432 : data loading time 0.00 +[Apr 27, 2026 - 19:35:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 432 : avg forward pass time 12.63 +[Apr 27, 2026 - 19:35:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 432 : backward pass time 11.59 +[Apr 27, 2026 - 19:35:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 432 : optimizer step time 1.07 +[Apr 27, 2026 - 19:35:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 432-------------------- +[Apr 27, 2026 - 19:35:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0176 iter count: 1.0 +[Apr 27, 2026 - 19:35:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0176 iter count: 1.0 +[Apr 27, 2026 - 19:35:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:35:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 19:35:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.085735 43.163082 42.987759 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.987305 70.058594 69.917969 +util 97.750000 99.000000 94.000000 +[Apr 27, 2026 - 19:35:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 433 : avg iteration time 47.62 seconds +[Apr 27, 2026 - 19:35:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 433 : data loading time 0.00 +[Apr 27, 2026 - 19:35:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 433 : avg forward pass time 11.06 +[Apr 27, 2026 - 19:35:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 433 : backward pass time 12.85 +[Apr 27, 2026 - 19:35:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 433 : optimizer step time 1.08 +[Apr 27, 2026 - 19:35:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 433-------------------- +[Apr 27, 2026 - 19:35:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0166 iter count: 1.0 +[Apr 27, 2026 - 19:35:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0166 iter count: 1.0 +[Apr 27, 2026 - 19:35:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:35:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 19:36:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.085604 43.163082 42.987759 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.982422 70.035156 69.937500 +util 97.500000 99.000000 93.000000 +[Apr 27, 2026 - 19:36:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 434 : avg iteration time 49.99 seconds +[Apr 27, 2026 - 19:36:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 434 : data loading time 0.00 +[Apr 27, 2026 - 19:36:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 434 : avg forward pass time 12.93 +[Apr 27, 2026 - 19:36:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 434 : backward pass time 11.55 +[Apr 27, 2026 - 19:36:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 434 : optimizer step time 1.01 +[Apr 27, 2026 - 19:36:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 434-------------------- +[Apr 27, 2026 - 19:36:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0200 iter count: 1.0 +[Apr 27, 2026 - 19:36:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0200 iter count: 1.0 +[Apr 27, 2026 - 19:36:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:36:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 19:39:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.085411 43.163082 42.987759 +peak_gpu_mem_gb 129.581032 131.620712 123.567759 +peak_gpu_mem_reserved_gb 133.594238 135.638672 127.507812 +util 69.000000 77.000000 63.000000 +[Apr 27, 2026 - 19:39:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 435 : avg iteration time 174.67 seconds +[Apr 27, 2026 - 19:39:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 435 : data loading time 0.00 +[Apr 27, 2026 - 19:39:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 435 : avg forward pass time 78.86 +[Apr 27, 2026 - 19:39:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 435 : backward pass time 8.34 +[Apr 27, 2026 - 19:39:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 435 : optimizer step time 0.33 +[Apr 27, 2026 - 19:39:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 435-------------------- +[Apr 27, 2026 - 19:39:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 833675.1543 iter count: 1.0 +[Apr 27, 2026 - 19:39:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6807 iter count: 1.0 +[Apr 27, 2026 - 19:39:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 833675.1543iter count: 1.0 +[Apr 27, 2026 - 19:39:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:39:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.3305iter count: 1.0 +[Apr 27, 2026 - 19:39:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.1344 iter count: 1.0 +[Apr 27, 2026 - 19:39:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.4899 iter count: 1.0 +[Apr 27, 2026 - 19:39:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 976570.5000iter count: 1.0 +[Apr 27, 2026 - 19:39:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 9.6681 iter count: 1.0 +[Apr 27, 2026 - 19:39:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 23671134.0000iter count: 1.0 +[Apr 27, 2026 - 19:39:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6807iter count: 1.0 +[Apr 27, 2026 - 19:39:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.0121 iter count: 1.0 +[Apr 27, 2026 - 19:39:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 3053.9087iter count: 1.0 +[Apr 27, 2026 - 19:39:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 1.4989 iter count: 1.0 +[Apr 27, 2026 - 19:39:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 20.0403 iter count: 1.0 +[Apr 27, 2026 - 19:39:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.3044 iter count: 1.0 +[Apr 27, 2026 - 19:39:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 40487.5352iter count: 1.0 +[Apr 27, 2026 - 19:39:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 2.3718 iter count: 1.0 +[Apr 27, 2026 - 19:39:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 114.8400iter count: 1.0 +[Apr 27, 2026 - 19:39:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0161 iter count: 1.0 +[Apr 27, 2026 - 19:39:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:39:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 19:40:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.085482 43.163090 42.987774 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.001953 70.175781 69.878906 +util 95.750000 99.000000 92.000000 +[Apr 27, 2026 - 19:40:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 436 : avg iteration time 52.63 seconds +[Apr 27, 2026 - 19:40:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 436 : data loading time 0.00 +[Apr 27, 2026 - 19:40:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 436 : avg forward pass time 13.56 +[Apr 27, 2026 - 19:40:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 436 : backward pass time 12.62 +[Apr 27, 2026 - 19:40:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 436 : optimizer step time 1.33 +[Apr 27, 2026 - 19:40:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 436-------------------- +[Apr 27, 2026 - 19:40:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0181 iter count: 1.0 +[Apr 27, 2026 - 19:40:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0181 iter count: 1.0 +[Apr 27, 2026 - 19:40:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:40:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 19:41:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.085470 43.163082 42.987759 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.948242 70.097656 69.820312 +util 96.500000 99.000000 91.000000 +[Apr 27, 2026 - 19:41:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 437 : avg iteration time 52.73 seconds +[Apr 27, 2026 - 19:41:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 437 : data loading time 0.00 +[Apr 27, 2026 - 19:41:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 437 : avg forward pass time 14.31 +[Apr 27, 2026 - 19:41:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 437 : backward pass time 11.54 +[Apr 27, 2026 - 19:41:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 437 : optimizer step time 1.00 +[Apr 27, 2026 - 19:41:21 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 437-------------------- +[Apr 27, 2026 - 19:41:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0190 iter count: 1.0 +[Apr 27, 2026 - 19:41:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0190 iter count: 1.0 +[Apr 27, 2026 - 19:41:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:41:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 19:42:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.085470 43.163082 42.987759 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.914062 70.097656 69.839844 +util 97.000000 99.000000 92.000000 +[Apr 27, 2026 - 19:42:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 438 : avg iteration time 47.81 seconds +[Apr 27, 2026 - 19:42:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 438 : data loading time 0.00 +[Apr 27, 2026 - 19:42:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 438 : avg forward pass time 11.42 +[Apr 27, 2026 - 19:42:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 438 : backward pass time 11.56 +[Apr 27, 2026 - 19:42:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 438 : optimizer step time 0.97 +[Apr 27, 2026 - 19:42:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 438-------------------- +[Apr 27, 2026 - 19:42:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0177 iter count: 1.0 +[Apr 27, 2026 - 19:42:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0177 iter count: 1.0 +[Apr 27, 2026 - 19:42:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:42:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 19:42:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.085716 43.163082 42.987759 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.923828 70.097656 69.820312 +util 97.500000 100.000000 92.000000 +[Apr 27, 2026 - 19:42:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 439 : avg iteration time 50.34 seconds +[Apr 27, 2026 - 19:42:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 439 : data loading time 0.00 +[Apr 27, 2026 - 19:42:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 439 : avg forward pass time 12.73 +[Apr 27, 2026 - 19:42:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 439 : backward pass time 11.58 +[Apr 27, 2026 - 19:42:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 439 : optimizer step time 1.32 +[Apr 27, 2026 - 19:42:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 439-------------------- +[Apr 27, 2026 - 19:42:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0165 iter count: 1.0 +[Apr 27, 2026 - 19:42:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0165 iter count: 1.0 +[Apr 27, 2026 - 19:42:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:42:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 19:45:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.085444 43.163082 42.987759 +peak_gpu_mem_gb 129.581032 131.620712 123.567759 +peak_gpu_mem_reserved_gb 133.564941 135.599609 127.468750 +util 68.500000 74.000000 64.000000 +[Apr 27, 2026 - 19:45:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 440 : avg iteration time 179.54 seconds +[Apr 27, 2026 - 19:45:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 440 : data loading time 0.00 +[Apr 27, 2026 - 19:45:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 440 : avg forward pass time 81.22 +[Apr 27, 2026 - 19:45:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 440 : backward pass time 8.41 +[Apr 27, 2026 - 19:45:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 440 : optimizer step time 0.33 +[Apr 27, 2026 - 19:45:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 440-------------------- +[Apr 27, 2026 - 19:45:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 202227.0107 iter count: 1.0 +[Apr 27, 2026 - 19:45:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6382 iter count: 1.0 +[Apr 27, 2026 - 19:45:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 202227.0107iter count: 1.0 +[Apr 27, 2026 - 19:45:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:45:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 3.5218iter count: 1.0 +[Apr 27, 2026 - 19:45:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.7729 iter count: 1.0 +[Apr 27, 2026 - 19:45:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.8804 iter count: 1.0 +[Apr 27, 2026 - 19:45:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 229025.6562iter count: 1.0 +[Apr 27, 2026 - 19:45:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 4.6917 iter count: 1.0 +[Apr 27, 2026 - 19:45:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 6995352.0000iter count: 1.0 +[Apr 27, 2026 - 19:45:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6382iter count: 1.0 +[Apr 27, 2026 - 19:45:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.8377 iter count: 1.0 +[Apr 27, 2026 - 19:45:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 15921.7549iter count: 1.0 +[Apr 27, 2026 - 19:45:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 2.7397 iter count: 1.0 +[Apr 27, 2026 - 19:45:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 239.7227iter count: 1.0 +[Apr 27, 2026 - 19:45:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.3326 iter count: 1.0 +[Apr 27, 2026 - 19:45:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 784.6438iter count: 1.0 +[Apr 27, 2026 - 19:45:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.1107 iter count: 1.0 +[Apr 27, 2026 - 19:45:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 27486.1992iter count: 1.0 +[Apr 27, 2026 - 19:45:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0192 iter count: 1.0 +[Apr 27, 2026 - 19:45:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:45:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 19:46:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.085513 43.163082 42.987759 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.889648 70.097656 69.742188 +util 98.500000 100.000000 96.000000 +[Apr 27, 2026 - 19:46:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 441 : avg iteration time 48.90 seconds +[Apr 27, 2026 - 19:46:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 441 : data loading time 0.00 +[Apr 27, 2026 - 19:46:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 441 : avg forward pass time 11.75 +[Apr 27, 2026 - 19:46:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 441 : backward pass time 11.57 +[Apr 27, 2026 - 19:46:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 441 : optimizer step time 0.98 +[Apr 27, 2026 - 19:46:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 441-------------------- +[Apr 27, 2026 - 19:46:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0219 iter count: 1.0 +[Apr 27, 2026 - 19:46:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0219 iter count: 1.0 +[Apr 27, 2026 - 19:46:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:46:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 19:47:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.085513 43.163082 42.987759 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 71.398438 72.730469 70.035156 +util 97.750000 100.000000 92.000000 +[Apr 27, 2026 - 19:47:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 442 : avg iteration time 51.04 seconds +[Apr 27, 2026 - 19:47:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 442 : data loading time 0.00 +[Apr 27, 2026 - 19:47:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 442 : avg forward pass time 12.79 +[Apr 27, 2026 - 19:47:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 442 : backward pass time 12.29 +[Apr 27, 2026 - 19:47:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 442 : optimizer step time 1.58 +[Apr 27, 2026 - 19:47:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 442-------------------- +[Apr 27, 2026 - 19:47:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0229 iter count: 1.0 +[Apr 27, 2026 - 19:47:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0229 iter count: 1.0 +[Apr 27, 2026 - 19:47:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:47:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 19:48:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.085519 43.163105 42.987774 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 71.359375 72.652344 69.898438 +util 96.500000 99.000000 93.000000 +[Apr 27, 2026 - 19:48:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 443 : avg iteration time 47.65 seconds +[Apr 27, 2026 - 19:48:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 443 : data loading time 0.00 +[Apr 27, 2026 - 19:48:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 443 : avg forward pass time 11.17 +[Apr 27, 2026 - 19:48:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 443 : backward pass time 12.71 +[Apr 27, 2026 - 19:48:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 443 : optimizer step time 1.04 +[Apr 27, 2026 - 19:48:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 443-------------------- +[Apr 27, 2026 - 19:48:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0198 iter count: 1.0 +[Apr 27, 2026 - 19:48:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0198 iter count: 1.0 +[Apr 27, 2026 - 19:48:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:48:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 19:49:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.085267 43.163082 42.987759 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.016602 70.195312 69.957031 +util 98.000000 99.000000 95.000000 +[Apr 27, 2026 - 19:49:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 444 : avg iteration time 49.29 seconds +[Apr 27, 2026 - 19:49:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 444 : data loading time 0.00 +[Apr 27, 2026 - 19:49:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 444 : avg forward pass time 12.59 +[Apr 27, 2026 - 19:49:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 444 : backward pass time 11.57 +[Apr 27, 2026 - 19:49:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 444 : optimizer step time 0.98 +[Apr 27, 2026 - 19:49:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 444-------------------- +[Apr 27, 2026 - 19:49:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0194 iter count: 1.0 +[Apr 27, 2026 - 19:49:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0194 iter count: 1.0 +[Apr 27, 2026 - 19:49:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:49:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 19:52:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.085267 43.163082 42.987759 +peak_gpu_mem_gb 129.581032 131.620712 123.567759 +peak_gpu_mem_reserved_gb 133.529785 135.560547 127.527344 +util 63.750000 67.000000 57.000000 +[Apr 27, 2026 - 19:52:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 445 : avg iteration time 177.94 seconds +[Apr 27, 2026 - 19:52:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 445 : data loading time 0.00 +[Apr 27, 2026 - 19:52:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 445 : avg forward pass time 79.88 +[Apr 27, 2026 - 19:52:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 445 : backward pass time 9.49 +[Apr 27, 2026 - 19:52:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 445 : optimizer step time 0.43 +[Apr 27, 2026 - 19:52:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 445-------------------- +[Apr 27, 2026 - 19:52:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 577725.0750 iter count: 1.0 +[Apr 27, 2026 - 19:52:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6260 iter count: 1.0 +[Apr 27, 2026 - 19:52:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 577725.0750iter count: 1.0 +[Apr 27, 2026 - 19:52:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:52:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.4658iter count: 1.0 +[Apr 27, 2026 - 19:52:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.6269 iter count: 1.0 +[Apr 27, 2026 - 19:52:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.2967 iter count: 1.0 +[Apr 27, 2026 - 19:52:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 955967.2500iter count: 1.0 +[Apr 27, 2026 - 19:52:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 3.5038 iter count: 1.0 +[Apr 27, 2026 - 19:52:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 16086616.0000iter count: 1.0 +[Apr 27, 2026 - 19:52:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6260iter count: 1.0 +[Apr 27, 2026 - 19:52:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.2944 iter count: 1.0 +[Apr 27, 2026 - 19:52:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 726.9222iter count: 1.0 +[Apr 27, 2026 - 19:52:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.5790 iter count: 1.0 +[Apr 27, 2026 - 19:52:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 70119.3594iter count: 1.0 +[Apr 27, 2026 - 19:52:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 2.8701 iter count: 1.0 +[Apr 27, 2026 - 19:52:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 311.1480iter count: 1.0 +[Apr 27, 2026 - 19:52:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.1038 iter count: 1.0 +[Apr 27, 2026 - 19:52:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 200322.8125iter count: 1.0 +[Apr 27, 2026 - 19:52:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0235 iter count: 1.0 +[Apr 27, 2026 - 19:52:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:52:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.02 seconds +[Apr 27, 2026 - 19:53:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.085267 43.163082 42.987759 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.909668 70.119141 69.820312 +util 95.500000 99.000000 91.000000 +[Apr 27, 2026 - 19:53:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 446 : avg iteration time 54.11 seconds +[Apr 27, 2026 - 19:53:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 446 : data loading time 0.00 +[Apr 27, 2026 - 19:53:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 446 : avg forward pass time 14.99 +[Apr 27, 2026 - 19:53:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 446 : backward pass time 11.57 +[Apr 27, 2026 - 19:53:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 446 : optimizer step time 0.99 +[Apr 27, 2026 - 19:53:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 446-------------------- +[Apr 27, 2026 - 19:53:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0227 iter count: 1.0 +[Apr 27, 2026 - 19:53:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0227 iter count: 1.0 +[Apr 27, 2026 - 19:53:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:53:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 19:53:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.085274 43.163086 42.987759 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.948730 70.099609 69.859375 +util 98.250000 100.000000 95.000000 +[Apr 27, 2026 - 19:53:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 447 : avg iteration time 50.35 seconds +[Apr 27, 2026 - 19:53:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 447 : data loading time 0.00 +[Apr 27, 2026 - 19:53:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 447 : avg forward pass time 12.56 +[Apr 27, 2026 - 19:53:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 447 : backward pass time 11.55 +[Apr 27, 2026 - 19:53:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 447 : optimizer step time 0.97 +[Apr 27, 2026 - 19:53:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 447-------------------- +[Apr 27, 2026 - 19:53:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0219 iter count: 1.0 +[Apr 27, 2026 - 19:53:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0219 iter count: 1.0 +[Apr 27, 2026 - 19:53:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:53:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 19:54:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.085269 43.163082 42.987759 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.948730 70.099609 69.859375 +util 95.500000 100.000000 90.000000 +[Apr 27, 2026 - 19:54:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 448 : avg iteration time 54.07 seconds +[Apr 27, 2026 - 19:54:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 448 : data loading time 0.00 +[Apr 27, 2026 - 19:54:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 448 : avg forward pass time 14.20 +[Apr 27, 2026 - 19:54:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 448 : backward pass time 12.83 +[Apr 27, 2026 - 19:54:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 448 : optimizer step time 1.26 +[Apr 27, 2026 - 19:54:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 448-------------------- +[Apr 27, 2026 - 19:54:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0264 iter count: 1.0 +[Apr 27, 2026 - 19:54:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0264 iter count: 1.0 +[Apr 27, 2026 - 19:54:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:54:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 19:55:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.085271 43.163082 42.987759 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.948730 70.099609 69.859375 +util 97.000000 100.000000 91.000000 +[Apr 27, 2026 - 19:55:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 449 : avg iteration time 50.48 seconds +[Apr 27, 2026 - 19:55:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 449 : data loading time 0.00 +[Apr 27, 2026 - 19:55:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 449 : avg forward pass time 12.65 +[Apr 27, 2026 - 19:55:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 449 : backward pass time 12.55 +[Apr 27, 2026 - 19:55:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 449 : optimizer step time 1.03 +[Apr 27, 2026 - 19:55:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 449-------------------- +[Apr 27, 2026 - 19:55:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0213 iter count: 1.0 +[Apr 27, 2026 - 19:55:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0213 iter count: 1.0 +[Apr 27, 2026 - 19:55:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:55:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 19:58:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.084981 43.163082 42.987778 +peak_gpu_mem_gb 129.581032 131.620712 123.567759 +peak_gpu_mem_reserved_gb 133.525879 135.619141 127.449219 +util 72.250000 77.000000 65.000000 +[Apr 27, 2026 - 19:58:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 450 : avg iteration time 174.85 seconds +[Apr 27, 2026 - 19:58:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 450 : data loading time 0.00 +[Apr 27, 2026 - 19:58:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 450 : avg forward pass time 78.92 +[Apr 27, 2026 - 19:58:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 450 : backward pass time 8.38 +[Apr 27, 2026 - 19:58:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 450 : optimizer step time 0.34 +[Apr 27, 2026 - 19:58:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 450-------------------- +[Apr 27, 2026 - 19:58:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 84058.3732 iter count: 1.0 +[Apr 27, 2026 - 19:58:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6155 iter count: 1.0 +[Apr 27, 2026 - 19:58:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 84058.3732iter count: 1.0 +[Apr 27, 2026 - 19:58:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:58:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.3831iter count: 1.0 +[Apr 27, 2026 - 19:58:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.9876 iter count: 1.0 +[Apr 27, 2026 - 19:58:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.1204 iter count: 1.0 +[Apr 27, 2026 - 19:58:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 106279.3984iter count: 1.0 +[Apr 27, 2026 - 19:58:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 7.2081 iter count: 1.0 +[Apr 27, 2026 - 19:58:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 1529986.3750iter count: 1.0 +[Apr 27, 2026 - 19:58:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6155iter count: 1.0 +[Apr 27, 2026 - 19:58:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.0008 iter count: 1.0 +[Apr 27, 2026 - 19:58:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 404.0903iter count: 1.0 +[Apr 27, 2026 - 19:58:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 2.9973 iter count: 1.0 +[Apr 27, 2026 - 19:58:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 401.2739iter count: 1.0 +[Apr 27, 2026 - 19:58:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.9384 iter count: 1.0 +[Apr 27, 2026 - 19:58:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 19472.5586iter count: 1.0 +[Apr 27, 2026 - 19:58:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.8075 iter count: 1.0 +[Apr 27, 2026 - 19:58:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 14988.9482iter count: 1.0 +[Apr 27, 2026 - 19:58:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0276 iter count: 1.0 +[Apr 27, 2026 - 19:58:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:58:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 19:59:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.084977 43.163082 42.987762 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.836426 69.937500 69.742188 +util 98.000000 99.000000 95.000000 +[Apr 27, 2026 - 19:59:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 451 : avg iteration time 54.60 seconds +[Apr 27, 2026 - 19:59:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 451 : data loading time 0.00 +[Apr 27, 2026 - 19:59:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 451 : avg forward pass time 14.67 +[Apr 27, 2026 - 19:59:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 451 : backward pass time 11.54 +[Apr 27, 2026 - 19:59:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 451 : optimizer step time 0.99 +[Apr 27, 2026 - 19:59:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 451-------------------- +[Apr 27, 2026 - 19:59:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0291 iter count: 1.0 +[Apr 27, 2026 - 19:59:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0291 iter count: 1.0 +[Apr 27, 2026 - 19:59:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 19:59:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 20:00:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.085281 43.163082 42.987759 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.875488 70.093750 69.689453 +util 96.750000 99.000000 91.000000 +[Apr 27, 2026 - 20:00:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 452 : avg iteration time 50.35 seconds +[Apr 27, 2026 - 20:00:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 452 : data loading time 0.00 +[Apr 27, 2026 - 20:00:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 452 : avg forward pass time 12.54 +[Apr 27, 2026 - 20:00:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 452 : backward pass time 12.68 +[Apr 27, 2026 - 20:00:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 452 : optimizer step time 1.04 +[Apr 27, 2026 - 20:00:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 452-------------------- +[Apr 27, 2026 - 20:00:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0282 iter count: 1.0 +[Apr 27, 2026 - 20:00:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0282 iter count: 1.0 +[Apr 27, 2026 - 20:00:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:00:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 20:01:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.085281 43.163082 42.987759 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.924316 70.074219 69.859375 +util 97.000000 99.000000 92.000000 +[Apr 27, 2026 - 20:01:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 453 : avg iteration time 51.68 seconds +[Apr 27, 2026 - 20:01:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 453 : data loading time 0.00 +[Apr 27, 2026 - 20:01:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 453 : avg forward pass time 13.78 +[Apr 27, 2026 - 20:01:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 453 : backward pass time 11.57 +[Apr 27, 2026 - 20:01:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 453 : optimizer step time 0.99 +[Apr 27, 2026 - 20:01:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 453-------------------- +[Apr 27, 2026 - 20:01:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0297 iter count: 1.0 +[Apr 27, 2026 - 20:01:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0297 iter count: 1.0 +[Apr 27, 2026 - 20:01:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:01:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 20:02:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.305708 43.365913 43.194241 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.909668 70.074219 69.845703 +util 98.250000 100.000000 94.000000 +[Apr 27, 2026 - 20:02:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 454 : avg iteration time 54.16 seconds +[Apr 27, 2026 - 20:02:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 454 : data loading time 0.00 +[Apr 27, 2026 - 20:02:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 454 : avg forward pass time 13.79 +[Apr 27, 2026 - 20:02:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 454 : backward pass time 12.17 +[Apr 27, 2026 - 20:02:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 454 : optimizer step time 1.57 +[Apr 27, 2026 - 20:02:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 454-------------------- +[Apr 27, 2026 - 20:02:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0260 iter count: 1.0 +[Apr 27, 2026 - 20:02:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0260 iter count: 1.0 +[Apr 27, 2026 - 20:02:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:02:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 20:04:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.305997 43.366600 43.194859 +peak_gpu_mem_gb 129.581032 131.620712 123.567759 +peak_gpu_mem_reserved_gb 133.492676 135.541016 127.511719 +util 73.750000 78.000000 67.000000 +[Apr 27, 2026 - 20:04:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 455 : avg iteration time 170.03 seconds +[Apr 27, 2026 - 20:04:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 455 : data loading time 0.00 +[Apr 27, 2026 - 20:04:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 455 : avg forward pass time 76.45 +[Apr 27, 2026 - 20:04:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 455 : backward pass time 8.40 +[Apr 27, 2026 - 20:04:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 455 : optimizer step time 0.34 +[Apr 27, 2026 - 20:04:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 455-------------------- +[Apr 27, 2026 - 20:04:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 188828.5564 iter count: 1.0 +[Apr 27, 2026 - 20:04:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6557 iter count: 1.0 +[Apr 27, 2026 - 20:04:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 188828.5564iter count: 1.0 +[Apr 27, 2026 - 20:04:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:04:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.1664iter count: 1.0 +[Apr 27, 2026 - 20:04:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.6247 iter count: 1.0 +[Apr 27, 2026 - 20:04:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.5615 iter count: 1.0 +[Apr 27, 2026 - 20:04:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 253712.5938iter count: 1.0 +[Apr 27, 2026 - 20:04:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 3.4879 iter count: 1.0 +[Apr 27, 2026 - 20:04:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 3696863.5000iter count: 1.0 +[Apr 27, 2026 - 20:04:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6557iter count: 1.0 +[Apr 27, 2026 - 20:04:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.6431 iter count: 1.0 +[Apr 27, 2026 - 20:04:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 79715.4375iter count: 1.0 +[Apr 27, 2026 - 20:04:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.0077 iter count: 1.0 +[Apr 27, 2026 - 20:04:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 3027.0120iter count: 1.0 +[Apr 27, 2026 - 20:04:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.7667 iter count: 1.0 +[Apr 27, 2026 - 20:04:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 13812.9971iter count: 1.0 +[Apr 27, 2026 - 20:04:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 0.6247 iter count: 1.0 +[Apr 27, 2026 - 20:04:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 3.4879 iter count: 1.0 +[Apr 27, 2026 - 20:04:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0222 iter count: 1.0 +[Apr 27, 2026 - 20:04:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:04:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 20:05:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.306538 43.366802 43.195065 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.973145 70.041016 69.937500 +util 97.750000 99.000000 95.000000 +[Apr 27, 2026 - 20:05:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 456 : avg iteration time 52.11 seconds +[Apr 27, 2026 - 20:05:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 456 : data loading time 0.00 +[Apr 27, 2026 - 20:05:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 456 : avg forward pass time 13.41 +[Apr 27, 2026 - 20:05:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 456 : backward pass time 11.80 +[Apr 27, 2026 - 20:05:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 456 : optimizer step time 0.98 +[Apr 27, 2026 - 20:05:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 456-------------------- +[Apr 27, 2026 - 20:05:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0299 iter count: 1.0 +[Apr 27, 2026 - 20:05:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0299 iter count: 1.0 +[Apr 27, 2026 - 20:05:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:05:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 20:06:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.307096 43.366943 43.195488 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.948730 70.060547 69.898438 +util 97.750000 100.000000 93.000000 +[Apr 27, 2026 - 20:06:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 457 : avg iteration time 54.25 seconds +[Apr 27, 2026 - 20:06:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 457 : data loading time 0.00 +[Apr 27, 2026 - 20:06:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 457 : avg forward pass time 14.27 +[Apr 27, 2026 - 20:06:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 457 : backward pass time 12.55 +[Apr 27, 2026 - 20:06:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 457 : optimizer step time 1.60 +[Apr 27, 2026 - 20:06:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 457-------------------- +[Apr 27, 2026 - 20:06:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0231 iter count: 1.0 +[Apr 27, 2026 - 20:06:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0231 iter count: 1.0 +[Apr 27, 2026 - 20:06:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:06:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 20:07:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.307654 43.367100 43.196560 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.987793 70.080078 69.937500 +util 96.250000 100.000000 92.000000 +[Apr 27, 2026 - 20:07:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 458 : avg iteration time 52.68 seconds +[Apr 27, 2026 - 20:07:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 458 : data loading time 0.00 +[Apr 27, 2026 - 20:07:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 458 : avg forward pass time 14.27 +[Apr 27, 2026 - 20:07:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 458 : backward pass time 11.56 +[Apr 27, 2026 - 20:07:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 458 : optimizer step time 0.99 +[Apr 27, 2026 - 20:07:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 458-------------------- +[Apr 27, 2026 - 20:07:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0278 iter count: 1.0 +[Apr 27, 2026 - 20:07:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0278 iter count: 1.0 +[Apr 27, 2026 - 20:07:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:07:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 20:08:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.307804 43.367279 43.196682 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.938965 70.060547 69.859375 +util 97.250000 100.000000 94.000000 +[Apr 27, 2026 - 20:08:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 459 : avg iteration time 53.05 seconds +[Apr 27, 2026 - 20:08:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 459 : data loading time 0.00 +[Apr 27, 2026 - 20:08:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 459 : avg forward pass time 14.11 +[Apr 27, 2026 - 20:08:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 459 : backward pass time 11.56 +[Apr 27, 2026 - 20:08:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 459 : optimizer step time 0.98 +[Apr 27, 2026 - 20:08:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 459-------------------- +[Apr 27, 2026 - 20:08:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0263 iter count: 1.0 +[Apr 27, 2026 - 20:08:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0263 iter count: 1.0 +[Apr 27, 2026 - 20:08:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:08:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 20:11:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.307825 43.368202 43.196781 +peak_gpu_mem_gb 129.581032 131.620712 123.567759 +peak_gpu_mem_reserved_gb 133.569824 135.599609 127.527344 +util 70.000000 74.000000 63.000000 +[Apr 27, 2026 - 20:11:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 460 : avg iteration time 173.68 seconds +[Apr 27, 2026 - 20:11:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 460 : data loading time 0.00 +[Apr 27, 2026 - 20:11:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 460 : avg forward pass time 78.32 +[Apr 27, 2026 - 20:11:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 460 : backward pass time 8.42 +[Apr 27, 2026 - 20:11:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 460 : optimizer step time 0.35 +[Apr 27, 2026 - 20:11:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 460-------------------- +[Apr 27, 2026 - 20:11:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 644750.7715 iter count: 1.0 +[Apr 27, 2026 - 20:11:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6293 iter count: 1.0 +[Apr 27, 2026 - 20:11:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 644750.7715iter count: 1.0 +[Apr 27, 2026 - 20:11:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:11:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.9982iter count: 1.0 +[Apr 27, 2026 - 20:11:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.6610 iter count: 1.0 +[Apr 27, 2026 - 20:11:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.0514 iter count: 1.0 +[Apr 27, 2026 - 20:11:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 874040.8125iter count: 1.0 +[Apr 27, 2026 - 20:11:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 3.7510 iter count: 1.0 +[Apr 27, 2026 - 20:11:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 9848324.0000iter count: 1.0 +[Apr 27, 2026 - 20:11:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6293iter count: 1.0 +[Apr 27, 2026 - 20:11:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 6.8930 iter count: 1.0 +[Apr 27, 2026 - 20:11:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 970936.6250iter count: 1.0 +[Apr 27, 2026 - 20:11:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.2009 iter count: 1.0 +[Apr 27, 2026 - 20:11:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 32916.3203iter count: 1.0 +[Apr 27, 2026 - 20:11:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.4227 iter count: 1.0 +[Apr 27, 2026 - 20:11:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 51301.9375iter count: 1.0 +[Apr 27, 2026 - 20:11:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 7.7960 iter count: 1.0 +[Apr 27, 2026 - 20:11:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 5909279.0000iter count: 1.0 +[Apr 27, 2026 - 20:11:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0253 iter count: 1.0 +[Apr 27, 2026 - 20:11:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:11:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 20:12:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.308379 43.368416 43.198616 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.943359 70.097656 69.800781 +util 97.000000 99.000000 94.000000 +[Apr 27, 2026 - 20:12:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 461 : avg iteration time 48.12 seconds +[Apr 27, 2026 - 20:12:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 461 : data loading time 0.00 +[Apr 27, 2026 - 20:12:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 461 : avg forward pass time 11.97 +[Apr 27, 2026 - 20:12:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 461 : backward pass time 11.58 +[Apr 27, 2026 - 20:12:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 461 : optimizer step time 0.97 +[Apr 27, 2026 - 20:12:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 461-------------------- +[Apr 27, 2026 - 20:12:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0269 iter count: 1.0 +[Apr 27, 2026 - 20:12:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0269 iter count: 1.0 +[Apr 27, 2026 - 20:12:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:12:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 20:13:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.308780 43.368557 43.198662 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.982422 70.054688 69.957031 +util 95.250000 100.000000 91.000000 +[Apr 27, 2026 - 20:13:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 462 : avg iteration time 47.77 seconds +[Apr 27, 2026 - 20:13:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 462 : data loading time 0.00 +[Apr 27, 2026 - 20:13:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 462 : avg forward pass time 11.14 +[Apr 27, 2026 - 20:13:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 462 : backward pass time 11.57 +[Apr 27, 2026 - 20:13:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 462 : optimizer step time 0.98 +[Apr 27, 2026 - 20:13:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 462-------------------- +[Apr 27, 2026 - 20:13:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0244 iter count: 1.0 +[Apr 27, 2026 - 20:13:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0244 iter count: 1.0 +[Apr 27, 2026 - 20:13:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:13:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 20:13:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.309062 43.368614 43.198639 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.982422 70.054688 69.957031 +util 99.000000 100.000000 97.000000 +[Apr 27, 2026 - 20:13:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 463 : avg iteration time 50.94 seconds +[Apr 27, 2026 - 20:13:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 463 : data loading time 0.00 +[Apr 27, 2026 - 20:13:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 463 : avg forward pass time 12.81 +[Apr 27, 2026 - 20:13:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 463 : backward pass time 12.02 +[Apr 27, 2026 - 20:13:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 463 : optimizer step time 1.56 +[Apr 27, 2026 - 20:13:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 463-------------------- +[Apr 27, 2026 - 20:13:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0291 iter count: 1.0 +[Apr 27, 2026 - 20:13:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0291 iter count: 1.0 +[Apr 27, 2026 - 20:13:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:13:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 20:14:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.309358 43.368690 43.199718 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.928711 70.035156 69.882812 +util 98.250000 99.000000 96.000000 +[Apr 27, 2026 - 20:14:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 464 : avg iteration time 50.32 seconds +[Apr 27, 2026 - 20:14:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 464 : data loading time 0.00 +[Apr 27, 2026 - 20:14:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 464 : avg forward pass time 12.67 +[Apr 27, 2026 - 20:14:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 464 : backward pass time 12.42 +[Apr 27, 2026 - 20:14:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 464 : optimizer step time 0.99 +[Apr 27, 2026 - 20:14:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 464-------------------- +[Apr 27, 2026 - 20:14:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0233 iter count: 1.0 +[Apr 27, 2026 - 20:14:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0233 iter count: 1.0 +[Apr 27, 2026 - 20:14:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:14:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 20:17:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.309229 43.368809 43.199696 +peak_gpu_mem_gb 129.581032 131.620712 123.567759 +peak_gpu_mem_reserved_gb 133.560547 135.580078 127.509766 +util 68.000000 74.000000 64.000000 +[Apr 27, 2026 - 20:17:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 465 : avg iteration time 174.37 seconds +[Apr 27, 2026 - 20:17:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 465 : data loading time 0.00 +[Apr 27, 2026 - 20:17:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 465 : avg forward pass time 78.66 +[Apr 27, 2026 - 20:17:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 465 : backward pass time 8.41 +[Apr 27, 2026 - 20:17:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 465 : optimizer step time 0.34 +[Apr 27, 2026 - 20:17:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 465-------------------- +[Apr 27, 2026 - 20:17:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 65535.2139 iter count: 1.0 +[Apr 27, 2026 - 20:17:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5281 iter count: 1.0 +[Apr 27, 2026 - 20:17:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 65535.2139iter count: 1.0 +[Apr 27, 2026 - 20:17:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:17:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.5516iter count: 1.0 +[Apr 27, 2026 - 20:17:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.7508 iter count: 1.0 +[Apr 27, 2026 - 20:17:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.0645 iter count: 1.0 +[Apr 27, 2026 - 20:17:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 127318.1562iter count: 1.0 +[Apr 27, 2026 - 20:17:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 4.4890 iter count: 1.0 +[Apr 27, 2026 - 20:17:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 1368082.3750iter count: 1.0 +[Apr 27, 2026 - 20:17:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5281iter count: 1.0 +[Apr 27, 2026 - 20:17:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 0.7508 iter count: 1.0 +[Apr 27, 2026 - 20:17:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 4.4890 iter count: 1.0 +[Apr 27, 2026 - 20:17:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.8081 iter count: 1.0 +[Apr 27, 2026 - 20:17:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 2030.6820iter count: 1.0 +[Apr 27, 2026 - 20:17:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.2003 iter count: 1.0 +[Apr 27, 2026 - 20:17:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 32879.5508iter count: 1.0 +[Apr 27, 2026 - 20:17:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.0179 iter count: 1.0 +[Apr 27, 2026 - 20:17:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 168684.7188iter count: 1.0 +[Apr 27, 2026 - 20:17:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0269 iter count: 1.0 +[Apr 27, 2026 - 20:17:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:17:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 20:18:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.309980 43.370705 43.199825 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.860840 69.923828 69.820312 +util 97.750000 99.000000 94.000000 +[Apr 27, 2026 - 20:18:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 466 : avg iteration time 49.48 seconds +[Apr 27, 2026 - 20:18:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 466 : data loading time 0.00 +[Apr 27, 2026 - 20:18:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 466 : avg forward pass time 11.98 +[Apr 27, 2026 - 20:18:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 466 : backward pass time 11.57 +[Apr 27, 2026 - 20:18:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 466 : optimizer step time 0.99 +[Apr 27, 2026 - 20:18:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 466-------------------- +[Apr 27, 2026 - 20:18:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0255 iter count: 1.0 +[Apr 27, 2026 - 20:18:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0255 iter count: 1.0 +[Apr 27, 2026 - 20:18:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:18:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 20:19:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.310026 43.370724 43.199917 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.841309 69.943359 69.781250 +util 97.250000 100.000000 91.000000 +[Apr 27, 2026 - 20:19:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 467 : avg iteration time 54.44 seconds +[Apr 27, 2026 - 20:19:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 467 : data loading time 0.00 +[Apr 27, 2026 - 20:19:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 467 : avg forward pass time 14.34 +[Apr 27, 2026 - 20:19:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 467 : backward pass time 12.73 +[Apr 27, 2026 - 20:19:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 467 : optimizer step time 1.47 +[Apr 27, 2026 - 20:19:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 467-------------------- +[Apr 27, 2026 - 20:19:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0276 iter count: 1.0 +[Apr 27, 2026 - 20:19:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0276 iter count: 1.0 +[Apr 27, 2026 - 20:19:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:19:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 20:20:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.310170 43.370724 43.200172 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.934082 70.015625 69.781250 +util 98.000000 99.000000 95.000000 +[Apr 27, 2026 - 20:20:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 468 : avg iteration time 52.77 seconds +[Apr 27, 2026 - 20:20:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 468 : data loading time 0.00 +[Apr 27, 2026 - 20:20:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 468 : avg forward pass time 14.33 +[Apr 27, 2026 - 20:20:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 468 : backward pass time 11.53 +[Apr 27, 2026 - 20:20:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 468 : optimizer step time 1.03 +[Apr 27, 2026 - 20:20:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 468-------------------- +[Apr 27, 2026 - 20:20:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0252 iter count: 1.0 +[Apr 27, 2026 - 20:20:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0252 iter count: 1.0 +[Apr 27, 2026 - 20:20:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:20:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 20:21:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.310422 43.370724 43.200279 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.953613 69.996094 69.904297 +util 97.500000 99.000000 93.000000 +[Apr 27, 2026 - 20:21:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 469 : avg iteration time 50.58 seconds +[Apr 27, 2026 - 20:21:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 469 : data loading time 0.00 +[Apr 27, 2026 - 20:21:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 469 : avg forward pass time 12.87 +[Apr 27, 2026 - 20:21:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 469 : backward pass time 11.56 +[Apr 27, 2026 - 20:21:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 469 : optimizer step time 0.98 +[Apr 27, 2026 - 20:21:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 469-------------------- +[Apr 27, 2026 - 20:21:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0258 iter count: 1.0 +[Apr 27, 2026 - 20:21:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0258 iter count: 1.0 +[Apr 27, 2026 - 20:21:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:21:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 20:23:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.310496 43.371719 43.200275 +peak_gpu_mem_gb 129.581032 131.620712 123.567759 +peak_gpu_mem_reserved_gb 133.511719 135.521484 127.490234 +util 72.250000 76.000000 65.000000 +[Apr 27, 2026 - 20:23:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 470 : avg iteration time 169.23 seconds +[Apr 27, 2026 - 20:23:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 470 : data loading time 0.00 +[Apr 27, 2026 - 20:23:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 470 : avg forward pass time 76.13 +[Apr 27, 2026 - 20:23:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 470 : backward pass time 8.34 +[Apr 27, 2026 - 20:23:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 470 : optimizer step time 0.34 +[Apr 27, 2026 - 20:23:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 470-------------------- +[Apr 27, 2026 - 20:23:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 271563.4688 iter count: 1.0 +[Apr 27, 2026 - 20:23:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6669 iter count: 1.0 +[Apr 27, 2026 - 20:23:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 271563.4688iter count: 1.0 +[Apr 27, 2026 - 20:23:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:23:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.9004iter count: 1.0 +[Apr 27, 2026 - 20:23:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.2365 iter count: 1.0 +[Apr 27, 2026 - 20:23:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.7517 iter count: 1.0 +[Apr 27, 2026 - 20:23:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 517913.7812iter count: 1.0 +[Apr 27, 2026 - 20:23:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 11.8585iter count: 1.0 +[Apr 27, 2026 - 20:23:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 5408466.5000iter count: 1.0 +[Apr 27, 2026 - 20:23:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6669iter count: 1.0 +[Apr 27, 2026 - 20:23:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 6.7550 iter count: 1.0 +[Apr 27, 2026 - 20:23:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 736799.3125iter count: 1.0 +[Apr 27, 2026 - 20:23:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.4874 iter count: 1.0 +[Apr 27, 2026 - 20:23:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 58383.1719iter count: 1.0 +[Apr 27, 2026 - 20:23:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 6.2035 iter count: 1.0 +[Apr 27, 2026 - 20:23:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 244526.4375iter count: 1.0 +[Apr 27, 2026 - 20:23:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 1.9171 iter count: 1.0 +[Apr 27, 2026 - 20:23:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 46.2597 iter count: 1.0 +[Apr 27, 2026 - 20:23:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0237 iter count: 1.0 +[Apr 27, 2026 - 20:23:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:23:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 20:24:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.310574 43.371742 43.200489 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.904785 69.923828 69.898438 +util 98.250000 100.000000 93.000000 +[Apr 27, 2026 - 20:24:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 471 : avg iteration time 50.17 seconds +[Apr 27, 2026 - 20:24:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 471 : data loading time 0.00 +[Apr 27, 2026 - 20:24:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 471 : avg forward pass time 12.24 +[Apr 27, 2026 - 20:24:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 471 : backward pass time 12.57 +[Apr 27, 2026 - 20:24:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 471 : optimizer step time 1.58 +[Apr 27, 2026 - 20:24:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 471-------------------- +[Apr 27, 2026 - 20:24:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0246 iter count: 1.0 +[Apr 27, 2026 - 20:24:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0246 iter count: 1.0 +[Apr 27, 2026 - 20:24:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:24:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 20:25:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.310619 43.371807 43.200607 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.895020 70.074219 69.820312 +util 98.000000 99.000000 95.000000 +[Apr 27, 2026 - 20:25:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 472 : avg iteration time 52.84 seconds +[Apr 27, 2026 - 20:25:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 472 : data loading time 0.00 +[Apr 27, 2026 - 20:25:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 472 : avg forward pass time 14.32 +[Apr 27, 2026 - 20:25:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 472 : backward pass time 11.61 +[Apr 27, 2026 - 20:25:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 472 : optimizer step time 1.01 +[Apr 27, 2026 - 20:25:35 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 472-------------------- +[Apr 27, 2026 - 20:25:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0239 iter count: 1.0 +[Apr 27, 2026 - 20:25:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0239 iter count: 1.0 +[Apr 27, 2026 - 20:25:35 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:25:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 20:26:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.311090 43.371807 43.200706 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.846191 69.917969 69.800781 +util 96.750000 99.000000 91.000000 +[Apr 27, 2026 - 20:26:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 473 : avg iteration time 50.35 seconds +[Apr 27, 2026 - 20:26:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 473 : data loading time 0.00 +[Apr 27, 2026 - 20:26:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 473 : avg forward pass time 12.78 +[Apr 27, 2026 - 20:26:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 473 : backward pass time 11.60 +[Apr 27, 2026 - 20:26:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 473 : optimizer step time 0.97 +[Apr 27, 2026 - 20:26:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 473-------------------- +[Apr 27, 2026 - 20:26:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0264 iter count: 1.0 +[Apr 27, 2026 - 20:26:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0264 iter count: 1.0 +[Apr 27, 2026 - 20:26:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:26:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 20:27:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.311154 43.371971 43.200848 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.929199 70.074219 69.845703 +util 96.250000 100.000000 91.000000 +[Apr 27, 2026 - 20:27:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 474 : avg iteration time 54.10 seconds +[Apr 27, 2026 - 20:27:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 474 : data loading time 0.00 +[Apr 27, 2026 - 20:27:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 474 : avg forward pass time 14.19 +[Apr 27, 2026 - 20:27:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 474 : backward pass time 12.52 +[Apr 27, 2026 - 20:27:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 474 : optimizer step time 1.60 +[Apr 27, 2026 - 20:27:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 474-------------------- +[Apr 27, 2026 - 20:27:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0262 iter count: 1.0 +[Apr 27, 2026 - 20:27:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0262 iter count: 1.0 +[Apr 27, 2026 - 20:27:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:27:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 20:30:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.311724 43.371971 43.200890 +peak_gpu_mem_gb 129.581032 131.620712 123.567759 +peak_gpu_mem_reserved_gb 133.570312 135.599609 127.529297 +util 70.000000 74.000000 63.000000 +[Apr 27, 2026 - 20:30:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 475 : avg iteration time 169.52 seconds +[Apr 27, 2026 - 20:30:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 475 : data loading time 0.00 +[Apr 27, 2026 - 20:30:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 475 : avg forward pass time 76.23 +[Apr 27, 2026 - 20:30:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 475 : backward pass time 8.37 +[Apr 27, 2026 - 20:30:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 475 : optimizer step time 0.35 +[Apr 27, 2026 - 20:30:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 475-------------------- +[Apr 27, 2026 - 20:30:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 514052.8135 iter count: 1.0 +[Apr 27, 2026 - 20:30:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6206 iter count: 1.0 +[Apr 27, 2026 - 20:30:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 514052.8135iter count: 1.0 +[Apr 27, 2026 - 20:30:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:30:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.3744iter count: 1.0 +[Apr 27, 2026 - 20:30:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.6917 iter count: 1.0 +[Apr 27, 2026 - 20:30:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.2624 iter count: 1.0 +[Apr 27, 2026 - 20:30:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 707889.9375iter count: 1.0 +[Apr 27, 2026 - 20:30:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 3.9882 iter count: 1.0 +[Apr 27, 2026 - 20:30:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 15018504.0000iter count: 1.0 +[Apr 27, 2026 - 20:30:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6206iter count: 1.0 +[Apr 27, 2026 - 20:30:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 6.1438 iter count: 1.0 +[Apr 27, 2026 - 20:30:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 216967.3906iter count: 1.0 +[Apr 27, 2026 - 20:30:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.1100 iter count: 1.0 +[Apr 27, 2026 - 20:30:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 3714.4502iter count: 1.0 +[Apr 27, 2026 - 20:30:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.2062 iter count: 1.0 +[Apr 27, 2026 - 20:30:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 609.3168iter count: 1.0 +[Apr 27, 2026 - 20:30:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.0602 iter count: 1.0 +[Apr 27, 2026 - 20:30:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 455.0404iter count: 1.0 +[Apr 27, 2026 - 20:30:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0238 iter count: 1.0 +[Apr 27, 2026 - 20:30:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:30:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 20:31:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.311893 43.372070 43.201180 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.958008 70.078125 69.917969 +util 99.000000 100.000000 97.000000 +[Apr 27, 2026 - 20:31:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 476 : avg iteration time 51.84 seconds +[Apr 27, 2026 - 20:31:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 476 : data loading time 0.00 +[Apr 27, 2026 - 20:31:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 476 : avg forward pass time 13.61 +[Apr 27, 2026 - 20:31:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 476 : backward pass time 12.07 +[Apr 27, 2026 - 20:31:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 476 : optimizer step time 0.99 +[Apr 27, 2026 - 20:31:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 476-------------------- +[Apr 27, 2026 - 20:31:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0222 iter count: 1.0 +[Apr 27, 2026 - 20:31:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0222 iter count: 1.0 +[Apr 27, 2026 - 20:31:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:31:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 20:31:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.311996 43.372265 43.201221 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.914062 69.960938 69.898438 +util 95.500000 99.000000 92.000000 +[Apr 27, 2026 - 20:31:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 477 : avg iteration time 49.27 seconds +[Apr 27, 2026 - 20:31:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 477 : data loading time 0.00 +[Apr 27, 2026 - 20:31:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 477 : avg forward pass time 12.57 +[Apr 27, 2026 - 20:31:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 477 : backward pass time 11.61 +[Apr 27, 2026 - 20:31:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 477 : optimizer step time 0.97 +[Apr 27, 2026 - 20:31:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 477-------------------- +[Apr 27, 2026 - 20:31:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0251 iter count: 1.0 +[Apr 27, 2026 - 20:31:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0251 iter count: 1.0 +[Apr 27, 2026 - 20:31:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:31:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 20:32:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.312016 43.372269 43.201271 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.923828 69.960938 69.898438 +util 99.000000 100.000000 97.000000 +[Apr 27, 2026 - 20:32:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 478 : avg iteration time 50.73 seconds +[Apr 27, 2026 - 20:32:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 478 : data loading time 0.00 +[Apr 27, 2026 - 20:32:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 478 : avg forward pass time 12.86 +[Apr 27, 2026 - 20:32:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 478 : backward pass time 11.86 +[Apr 27, 2026 - 20:32:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 478 : optimizer step time 1.52 +[Apr 27, 2026 - 20:32:41 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 478-------------------- +[Apr 27, 2026 - 20:32:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0220 iter count: 1.0 +[Apr 27, 2026 - 20:32:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0220 iter count: 1.0 +[Apr 27, 2026 - 20:32:41 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:32:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 20:33:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.312045 43.372334 43.201286 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.889648 69.960938 69.859375 +util 97.250000 99.000000 92.000000 +[Apr 27, 2026 - 20:33:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 479 : avg iteration time 52.82 seconds +[Apr 27, 2026 - 20:33:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 479 : data loading time 0.00 +[Apr 27, 2026 - 20:33:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 479 : avg forward pass time 14.33 +[Apr 27, 2026 - 20:33:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 479 : backward pass time 11.59 +[Apr 27, 2026 - 20:33:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 479 : optimizer step time 0.97 +[Apr 27, 2026 - 20:33:34 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 479-------------------- +[Apr 27, 2026 - 20:33:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0227 iter count: 1.0 +[Apr 27, 2026 - 20:33:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0227 iter count: 1.0 +[Apr 27, 2026 - 20:33:34 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:33:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 20:36:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.311789 43.372345 43.201336 +peak_gpu_mem_gb 129.581032 131.620712 123.567759 +peak_gpu_mem_reserved_gb 133.574707 135.619141 127.566406 +util 70.750000 77.000000 67.000000 +[Apr 27, 2026 - 20:36:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 480 : avg iteration time 175.09 seconds +[Apr 27, 2026 - 20:36:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 480 : data loading time 0.00 +[Apr 27, 2026 - 20:36:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 480 : avg forward pass time 79.03 +[Apr 27, 2026 - 20:36:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 480 : backward pass time 8.39 +[Apr 27, 2026 - 20:36:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 480 : optimizer step time 0.33 +[Apr 27, 2026 - 20:36:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 480-------------------- +[Apr 27, 2026 - 20:36:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 196472.7499 iter count: 1.0 +[Apr 27, 2026 - 20:36:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6036 iter count: 1.0 +[Apr 27, 2026 - 20:36:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 196472.7499iter count: 1.0 +[Apr 27, 2026 - 20:36:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:36:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.6825iter count: 1.0 +[Apr 27, 2026 - 20:36:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.4077 iter count: 1.0 +[Apr 27, 2026 - 20:36:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.9706 iter count: 1.0 +[Apr 27, 2026 - 20:36:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 396360.9688iter count: 1.0 +[Apr 27, 2026 - 20:36:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 2.2602 iter count: 1.0 +[Apr 27, 2026 - 20:36:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 8378410.5000iter count: 1.0 +[Apr 27, 2026 - 20:36:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6036iter count: 1.0 +[Apr 27, 2026 - 20:36:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.0771 iter count: 1.0 +[Apr 27, 2026 - 20:36:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 3478.0276iter count: 1.0 +[Apr 27, 2026 - 20:36:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 6.1699 iter count: 1.0 +[Apr 27, 2026 - 20:36:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 228608.7812iter count: 1.0 +[Apr 27, 2026 - 20:36:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.3149 iter count: 1.0 +[Apr 27, 2026 - 20:36:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 5595.9316iter count: 1.0 +[Apr 27, 2026 - 20:36:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.5643 iter count: 1.0 +[Apr 27, 2026 - 20:36:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 503100.7812iter count: 1.0 +[Apr 27, 2026 - 20:36:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0245 iter count: 1.0 +[Apr 27, 2026 - 20:36:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:36:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 20:37:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.311836 43.372417 43.201313 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.923828 69.980469 69.878906 +util 98.000000 99.000000 95.000000 +[Apr 27, 2026 - 20:37:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 481 : avg iteration time 51.87 seconds +[Apr 27, 2026 - 20:37:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 481 : data loading time 0.00 +[Apr 27, 2026 - 20:37:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 481 : avg forward pass time 13.23 +[Apr 27, 2026 - 20:37:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 481 : backward pass time 11.57 +[Apr 27, 2026 - 20:37:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 481 : optimizer step time 1.01 +[Apr 27, 2026 - 20:37:21 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 481-------------------- +[Apr 27, 2026 - 20:37:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0232 iter count: 1.0 +[Apr 27, 2026 - 20:37:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0232 iter count: 1.0 +[Apr 27, 2026 - 20:37:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:37:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 20:38:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.312190 43.372551 43.201614 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.840820 69.937500 69.746094 +util 94.250000 99.000000 91.000000 +[Apr 27, 2026 - 20:38:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 482 : avg iteration time 53.07 seconds +[Apr 27, 2026 - 20:38:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 482 : data loading time 0.00 +[Apr 27, 2026 - 20:38:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 482 : avg forward pass time 14.40 +[Apr 27, 2026 - 20:38:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 482 : backward pass time 11.68 +[Apr 27, 2026 - 20:38:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 482 : optimizer step time 0.97 +[Apr 27, 2026 - 20:38:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 482-------------------- +[Apr 27, 2026 - 20:38:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0200 iter count: 1.0 +[Apr 27, 2026 - 20:38:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0200 iter count: 1.0 +[Apr 27, 2026 - 20:38:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:38:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 20:39:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.312214 43.372524 43.201637 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.943359 69.960938 69.937500 +util 98.250000 100.000000 95.000000 +[Apr 27, 2026 - 20:39:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 483 : avg iteration time 53.13 seconds +[Apr 27, 2026 - 20:39:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 483 : data loading time 0.00 +[Apr 27, 2026 - 20:39:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 483 : avg forward pass time 14.19 +[Apr 27, 2026 - 20:39:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 483 : backward pass time 11.61 +[Apr 27, 2026 - 20:39:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 483 : optimizer step time 0.99 +[Apr 27, 2026 - 20:39:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 483-------------------- +[Apr 27, 2026 - 20:39:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0209 iter count: 1.0 +[Apr 27, 2026 - 20:39:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0209 iter count: 1.0 +[Apr 27, 2026 - 20:39:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:39:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 20:39:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.312275 43.372528 43.201637 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.884766 69.960938 69.820312 +util 97.750000 99.000000 94.000000 +[Apr 27, 2026 - 20:39:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 484 : avg iteration time 47.58 seconds +[Apr 27, 2026 - 20:39:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 484 : data loading time 0.00 +[Apr 27, 2026 - 20:39:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 484 : avg forward pass time 11.33 +[Apr 27, 2026 - 20:39:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 484 : backward pass time 11.55 +[Apr 27, 2026 - 20:39:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 484 : optimizer step time 1.07 +[Apr 27, 2026 - 20:39:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 484-------------------- +[Apr 27, 2026 - 20:39:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0203 iter count: 1.0 +[Apr 27, 2026 - 20:39:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0203 iter count: 1.0 +[Apr 27, 2026 - 20:39:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:39:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 20:42:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.312002 43.372620 43.201641 +peak_gpu_mem_gb 129.581032 131.620712 123.567759 +peak_gpu_mem_reserved_gb 133.506836 135.560547 127.431641 +util 65.250000 71.000000 61.000000 +[Apr 27, 2026 - 20:42:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 485 : avg iteration time 174.09 seconds +[Apr 27, 2026 - 20:42:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 485 : data loading time 0.00 +[Apr 27, 2026 - 20:42:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 485 : avg forward pass time 78.48 +[Apr 27, 2026 - 20:42:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 485 : backward pass time 8.38 +[Apr 27, 2026 - 20:42:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 485 : optimizer step time 0.34 +[Apr 27, 2026 - 20:42:49 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 485-------------------- +[Apr 27, 2026 - 20:42:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 152691.4395 iter count: 1.0 +[Apr 27, 2026 - 20:42:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6512 iter count: 1.0 +[Apr 27, 2026 - 20:42:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 152691.4395iter count: 1.0 +[Apr 27, 2026 - 20:42:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:42:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.4160iter count: 1.0 +[Apr 27, 2026 - 20:42:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.0987 iter count: 1.0 +[Apr 27, 2026 - 20:42:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.5172 iter count: 1.0 +[Apr 27, 2026 - 20:42:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 178098.1562iter count: 1.0 +[Apr 27, 2026 - 20:42:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 9.0016 iter count: 1.0 +[Apr 27, 2026 - 20:42:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 3383254.0000iter count: 1.0 +[Apr 27, 2026 - 20:42:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6512iter count: 1.0 +[Apr 27, 2026 - 20:42:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 1.2141 iter count: 1.0 +[Apr 27, 2026 - 20:42:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 11.3374 iter count: 1.0 +[Apr 27, 2026 - 20:42:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.6254 iter count: 1.0 +[Apr 27, 2026 - 20:42:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 76934.0469iter count: 1.0 +[Apr 27, 2026 - 20:42:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 6.0026 iter count: 1.0 +[Apr 27, 2026 - 20:42:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 163616.6562iter count: 1.0 +[Apr 27, 2026 - 20:42:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.9770 iter count: 1.0 +[Apr 27, 2026 - 20:42:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 155441.0000iter count: 1.0 +[Apr 27, 2026 - 20:42:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0215 iter count: 1.0 +[Apr 27, 2026 - 20:42:49 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:42:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 20:43:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.312014 43.372643 43.201622 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.938477 70.097656 69.781250 +util 94.500000 99.000000 92.000000 +[Apr 27, 2026 - 20:43:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 486 : avg iteration time 49.79 seconds +[Apr 27, 2026 - 20:43:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 486 : data loading time 0.00 +[Apr 27, 2026 - 20:43:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 486 : avg forward pass time 12.09 +[Apr 27, 2026 - 20:43:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 486 : backward pass time 11.54 +[Apr 27, 2026 - 20:43:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 486 : optimizer step time 1.00 +[Apr 27, 2026 - 20:43:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 486-------------------- +[Apr 27, 2026 - 20:43:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0183 iter count: 1.0 +[Apr 27, 2026 - 20:43:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0183 iter count: 1.0 +[Apr 27, 2026 - 20:43:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:43:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 20:44:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.312283 43.372654 43.201622 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.962891 70.175781 69.781250 +util 98.500000 100.000000 95.000000 +[Apr 27, 2026 - 20:44:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 487 : avg iteration time 54.54 seconds +[Apr 27, 2026 - 20:44:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 487 : data loading time 0.00 +[Apr 27, 2026 - 20:44:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 487 : avg forward pass time 14.37 +[Apr 27, 2026 - 20:44:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 487 : backward pass time 12.47 +[Apr 27, 2026 - 20:44:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 487 : optimizer step time 1.59 +[Apr 27, 2026 - 20:44:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 487-------------------- +[Apr 27, 2026 - 20:44:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0192 iter count: 1.0 +[Apr 27, 2026 - 20:44:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0192 iter count: 1.0 +[Apr 27, 2026 - 20:44:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:44:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 20:45:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.312265 43.372643 43.201603 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.904297 70.097656 69.781250 +util 99.000000 100.000000 98.000000 +[Apr 27, 2026 - 20:45:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 488 : avg iteration time 50.63 seconds +[Apr 27, 2026 - 20:45:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 488 : data loading time 0.00 +[Apr 27, 2026 - 20:45:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 488 : avg forward pass time 12.80 +[Apr 27, 2026 - 20:45:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 488 : backward pass time 12.39 +[Apr 27, 2026 - 20:45:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 488 : optimizer step time 1.02 +[Apr 27, 2026 - 20:45:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 488-------------------- +[Apr 27, 2026 - 20:45:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0173 iter count: 1.0 +[Apr 27, 2026 - 20:45:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0173 iter count: 1.0 +[Apr 27, 2026 - 20:45:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:45:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 20:46:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.312277 43.372627 43.201607 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.948242 70.175781 69.781250 +util 96.250000 99.000000 90.000000 +[Apr 27, 2026 - 20:46:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 489 : avg iteration time 49.31 seconds +[Apr 27, 2026 - 20:46:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 489 : data loading time 0.00 +[Apr 27, 2026 - 20:46:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 489 : avg forward pass time 12.58 +[Apr 27, 2026 - 20:46:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 489 : backward pass time 11.57 +[Apr 27, 2026 - 20:46:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 489 : optimizer step time 1.00 +[Apr 27, 2026 - 20:46:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 489-------------------- +[Apr 27, 2026 - 20:46:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0207 iter count: 1.0 +[Apr 27, 2026 - 20:46:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0207 iter count: 1.0 +[Apr 27, 2026 - 20:46:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:46:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 20:49:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.312215 43.372669 43.201637 +peak_gpu_mem_gb 129.581032 131.620712 123.567759 +peak_gpu_mem_reserved_gb 133.746582 135.599609 128.351562 +util 67.750000 73.000000 65.000000 +[Apr 27, 2026 - 20:49:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 490 : avg iteration time 179.41 seconds +[Apr 27, 2026 - 20:49:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 490 : data loading time 0.00 +[Apr 27, 2026 - 20:49:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 490 : avg forward pass time 80.57 +[Apr 27, 2026 - 20:49:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 490 : backward pass time 9.59 +[Apr 27, 2026 - 20:49:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 490 : optimizer step time 0.37 +[Apr 27, 2026 - 20:49:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 490-------------------- +[Apr 27, 2026 - 20:49:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 108428.8457 iter count: 1.0 +[Apr 27, 2026 - 20:49:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5738 iter count: 1.0 +[Apr 27, 2026 - 20:49:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 108428.8457iter count: 1.0 +[Apr 27, 2026 - 20:49:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:49:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.7108iter count: 1.0 +[Apr 27, 2026 - 20:49:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.5656 iter count: 1.0 +[Apr 27, 2026 - 20:49:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.3900 iter count: 1.0 +[Apr 27, 2026 - 20:49:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 204821.6406iter count: 1.0 +[Apr 27, 2026 - 20:49:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 22.8993iter count: 1.0 +[Apr 27, 2026 - 20:49:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 2623262.0000iter count: 1.0 +[Apr 27, 2026 - 20:49:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5738iter count: 1.0 +[Apr 27, 2026 - 20:49:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 6.3683 iter count: 1.0 +[Apr 27, 2026 - 20:49:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 339982.5312iter count: 1.0 +[Apr 27, 2026 - 20:49:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 6.0402 iter count: 1.0 +[Apr 27, 2026 - 20:49:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 176384.8125iter count: 1.0 +[Apr 27, 2026 - 20:49:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.8345 iter count: 1.0 +[Apr 27, 2026 - 20:49:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 116901.6641iter count: 1.0 +[Apr 27, 2026 - 20:49:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.1005 iter count: 1.0 +[Apr 27, 2026 - 20:49:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 198983.9062iter count: 1.0 +[Apr 27, 2026 - 20:49:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0205 iter count: 1.0 +[Apr 27, 2026 - 20:49:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:49:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 20:50:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.312255 43.372688 43.201645 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.967773 70.058594 69.917969 +util 97.750000 99.000000 94.000000 +[Apr 27, 2026 - 20:50:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 491 : avg iteration time 52.04 seconds +[Apr 27, 2026 - 20:50:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 491 : data loading time 0.00 +[Apr 27, 2026 - 20:50:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 491 : avg forward pass time 13.96 +[Apr 27, 2026 - 20:50:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 491 : backward pass time 11.54 +[Apr 27, 2026 - 20:50:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 491 : optimizer step time 1.01 +[Apr 27, 2026 - 20:50:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 491-------------------- +[Apr 27, 2026 - 20:50:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0221 iter count: 1.0 +[Apr 27, 2026 - 20:50:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0221 iter count: 1.0 +[Apr 27, 2026 - 20:50:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:50:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 20:50:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.312628 43.372768 43.201916 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.933594 70.039062 69.878906 +util 98.250000 100.000000 96.000000 +[Apr 27, 2026 - 20:50:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 492 : avg iteration time 50.54 seconds +[Apr 27, 2026 - 20:50:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 492 : data loading time 0.00 +[Apr 27, 2026 - 20:50:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 492 : avg forward pass time 12.80 +[Apr 27, 2026 - 20:50:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 492 : backward pass time 11.58 +[Apr 27, 2026 - 20:50:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 492 : optimizer step time 0.98 +[Apr 27, 2026 - 20:50:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 492-------------------- +[Apr 27, 2026 - 20:50:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0198 iter count: 1.0 +[Apr 27, 2026 - 20:50:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0198 iter count: 1.0 +[Apr 27, 2026 - 20:50:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:50:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 20:51:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.312606 43.372746 43.201920 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.948242 70.058594 69.898438 +util 97.000000 100.000000 91.000000 +[Apr 27, 2026 - 20:51:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 493 : avg iteration time 50.82 seconds +[Apr 27, 2026 - 20:51:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 493 : data loading time 0.00 +[Apr 27, 2026 - 20:51:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 493 : avg forward pass time 12.74 +[Apr 27, 2026 - 20:51:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 493 : backward pass time 11.61 +[Apr 27, 2026 - 20:51:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 493 : optimizer step time 1.35 +[Apr 27, 2026 - 20:51:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 493-------------------- +[Apr 27, 2026 - 20:51:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0242 iter count: 1.0 +[Apr 27, 2026 - 20:51:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0242 iter count: 1.0 +[Apr 27, 2026 - 20:51:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:51:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 20:52:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.312654 43.372746 43.202030 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.967773 70.058594 69.937500 +util 96.750000 99.000000 91.000000 +[Apr 27, 2026 - 20:52:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 494 : avg iteration time 50.80 seconds +[Apr 27, 2026 - 20:52:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 494 : data loading time 0.00 +[Apr 27, 2026 - 20:52:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 494 : avg forward pass time 12.78 +[Apr 27, 2026 - 20:52:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 494 : backward pass time 12.59 +[Apr 27, 2026 - 20:52:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 494 : optimizer step time 1.00 +[Apr 27, 2026 - 20:52:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 494-------------------- +[Apr 27, 2026 - 20:52:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0206 iter count: 1.0 +[Apr 27, 2026 - 20:52:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0206 iter count: 1.0 +[Apr 27, 2026 - 20:52:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:52:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 20:55:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.312493 43.372879 43.202057 +peak_gpu_mem_gb 129.581032 131.620712 123.567759 +peak_gpu_mem_reserved_gb 134.132324 135.615234 129.777344 +util 71.750000 79.000000 62.000000 +[Apr 27, 2026 - 20:55:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 495 : avg iteration time 175.70 seconds +[Apr 27, 2026 - 20:55:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 495 : data loading time 0.00 +[Apr 27, 2026 - 20:55:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 495 : avg forward pass time 79.34 +[Apr 27, 2026 - 20:55:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 495 : backward pass time 8.37 +[Apr 27, 2026 - 20:55:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 495 : optimizer step time 0.34 +[Apr 27, 2026 - 20:55:32 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 495-------------------- +[Apr 27, 2026 - 20:55:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 195747.1533 iter count: 1.0 +[Apr 27, 2026 - 20:55:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6490 iter count: 1.0 +[Apr 27, 2026 - 20:55:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 195747.1533iter count: 1.0 +[Apr 27, 2026 - 20:55:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:55:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.7293iter count: 1.0 +[Apr 27, 2026 - 20:55:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.8093 iter count: 1.0 +[Apr 27, 2026 - 20:55:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.5397 iter count: 1.0 +[Apr 27, 2026 - 20:55:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 251189.0469iter count: 1.0 +[Apr 27, 2026 - 20:55:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 37.2828iter count: 1.0 +[Apr 27, 2026 - 20:55:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 3539126.7500iter count: 1.0 +[Apr 27, 2026 - 20:55:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6490iter count: 1.0 +[Apr 27, 2026 - 20:55:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.6584 iter count: 1.0 +[Apr 27, 2026 - 20:55:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 82185.4375iter count: 1.0 +[Apr 27, 2026 - 20:55:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.8896 iter count: 1.0 +[Apr 27, 2026 - 20:55:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 17663.6562iter count: 1.0 +[Apr 27, 2026 - 20:55:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.9887 iter count: 1.0 +[Apr 27, 2026 - 20:55:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 159123.2188iter count: 1.0 +[Apr 27, 2026 - 20:55:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.8146 iter count: 1.0 +[Apr 27, 2026 - 20:55:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 2057.3013iter count: 1.0 +[Apr 27, 2026 - 20:55:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0190 iter count: 1.0 +[Apr 27, 2026 - 20:55:32 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:55:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 20:56:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.312566 43.372959 43.202045 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.880371 70.152344 69.494141 +util 97.250000 100.000000 95.000000 +[Apr 27, 2026 - 20:56:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 496 : avg iteration time 52.87 seconds +[Apr 27, 2026 - 20:56:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 496 : data loading time 0.00 +[Apr 27, 2026 - 20:56:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 496 : avg forward pass time 13.61 +[Apr 27, 2026 - 20:56:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 496 : backward pass time 11.63 +[Apr 27, 2026 - 20:56:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 496 : optimizer step time 0.98 +[Apr 27, 2026 - 20:56:25 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 496-------------------- +[Apr 27, 2026 - 20:56:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0254 iter count: 1.0 +[Apr 27, 2026 - 20:56:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0254 iter count: 1.0 +[Apr 27, 2026 - 20:56:25 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:56:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 20:57:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.312640 43.372967 43.202065 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.836426 69.976562 69.455078 +util 96.250000 100.000000 92.000000 +[Apr 27, 2026 - 20:57:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 497 : avg iteration time 47.97 seconds +[Apr 27, 2026 - 20:57:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 497 : data loading time 0.00 +[Apr 27, 2026 - 20:57:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 497 : avg forward pass time 11.55 +[Apr 27, 2026 - 20:57:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 497 : backward pass time 11.69 +[Apr 27, 2026 - 20:57:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 497 : optimizer step time 1.42 +[Apr 27, 2026 - 20:57:13 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 497-------------------- +[Apr 27, 2026 - 20:57:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0184 iter count: 1.0 +[Apr 27, 2026 - 20:57:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0184 iter count: 1.0 +[Apr 27, 2026 - 20:57:13 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:57:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 20:58:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.312637 43.372959 43.202080 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.860840 70.113281 69.455078 +util 97.500000 100.000000 95.000000 +[Apr 27, 2026 - 20:58:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 498 : avg iteration time 50.64 seconds +[Apr 27, 2026 - 20:58:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 498 : data loading time 0.00 +[Apr 27, 2026 - 20:58:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 498 : avg forward pass time 12.71 +[Apr 27, 2026 - 20:58:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 498 : backward pass time 12.61 +[Apr 27, 2026 - 20:58:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 498 : optimizer step time 1.03 +[Apr 27, 2026 - 20:58:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 498-------------------- +[Apr 27, 2026 - 20:58:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0220 iter count: 1.0 +[Apr 27, 2026 - 20:58:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0220 iter count: 1.0 +[Apr 27, 2026 - 20:58:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:58:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 20:58:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.312748 43.373077 43.202084 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.787598 69.937500 69.455078 +util 95.000000 99.000000 92.000000 +[Apr 27, 2026 - 20:58:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 499 : avg iteration time 50.17 seconds +[Apr 27, 2026 - 20:58:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 499 : data loading time 0.00 +[Apr 27, 2026 - 20:58:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 499 : avg forward pass time 13.03 +[Apr 27, 2026 - 20:58:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 499 : backward pass time 11.54 +[Apr 27, 2026 - 20:58:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 499 : optimizer step time 0.99 +[Apr 27, 2026 - 20:58:54 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 499-------------------- +[Apr 27, 2026 - 20:58:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0229 iter count: 1.0 +[Apr 27, 2026 - 20:58:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0229 iter count: 1.0 +[Apr 27, 2026 - 20:58:54 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 20:58:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 21:01:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.312546 43.373081 43.202087 +peak_gpu_mem_gb 129.581032 131.620712 123.567759 +peak_gpu_mem_reserved_gb 134.044434 135.556641 129.542969 +util 67.750000 71.000000 61.000000 +[Apr 27, 2026 - 21:01:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 500 : avg iteration time 177.87 seconds +[Apr 27, 2026 - 21:01:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 500 : data loading time 0.00 +[Apr 27, 2026 - 21:01:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 500 : avg forward pass time 80.39 +[Apr 27, 2026 - 21:01:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 500 : backward pass time 8.42 +[Apr 27, 2026 - 21:01:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 500 : optimizer step time 0.35 +[Apr 27, 2026 - 21:01:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 500-------------------- +[Apr 27, 2026 - 21:01:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 365603.1064 iter count: 1.0 +[Apr 27, 2026 - 21:01:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6481 iter count: 1.0 +[Apr 27, 2026 - 21:01:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 365603.1064iter count: 1.0 +[Apr 27, 2026 - 21:01:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 21:01:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.4968iter count: 1.0 +[Apr 27, 2026 - 21:01:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.1119 iter count: 1.0 +[Apr 27, 2026 - 21:01:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.1116 iter count: 1.0 +[Apr 27, 2026 - 21:01:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 489296.4062iter count: 1.0 +[Apr 27, 2026 - 21:01:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 9.2423 iter count: 1.0 +[Apr 27, 2026 - 21:01:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 11107967.0000iter count: 1.0 +[Apr 27, 2026 - 21:01:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6481iter count: 1.0 +[Apr 27, 2026 - 21:01:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 2.7859 iter count: 1.0 +[Apr 27, 2026 - 21:01:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 262.9120iter count: 1.0 +[Apr 27, 2026 - 21:01:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 7.4050 iter count: 1.0 +[Apr 27, 2026 - 21:01:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 2703092.5000iter count: 1.0 +[Apr 27, 2026 - 21:01:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 2.0400 iter count: 1.0 +[Apr 27, 2026 - 21:01:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 59.1505 iter count: 1.0 +[Apr 27, 2026 - 21:01:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.2961 iter count: 1.0 +[Apr 27, 2026 - 21:01:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 729.3707iter count: 1.0 +[Apr 27, 2026 - 21:01:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0221 iter count: 1.0 +[Apr 27, 2026 - 21:01:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[Apr 27, 2026 - 21:02:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 15.03 seconds +[Apr 27, 2026 - 21:02:07 | INFO | fastgen.trainer:save_checkpoint:266 ] Saving checkpoint iteration 500 +[Apr 27, 2026 - 21:02:07 | INFO | fastgen.utils.checkpointer:save:301 ] Saving FSDP model to prefix /tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000500 +[Apr 27, 2026 - 21:02:23 | SUCCESS | fastgen.utils.checkpointer:save:329 ] Model saved at iteration 500 +[Apr 27, 2026 - 21:02:23 | INFO | fastgen.trainer:validate:358 ] Validation iteration 500 +[Apr 27, 2026 - 21:02:24 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 21:02:26 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 2.2s +[Apr 27, 2026 - 21:02:29 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 0: sync_c=6.347 +[Apr 27, 2026 - 21:02:29 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 21:02:31 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.8s +[Apr 27, 2026 - 21:02:34 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 1: sync_c=4.978 +[Apr 27, 2026 - 21:02:35 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 21:02:36 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.8s +[Apr 27, 2026 - 21:02:39 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 2: sync_c=4.361 +[Apr 27, 2026 - 21:02:40 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 21:02:41 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.8s +[Apr 27, 2026 - 21:02:44 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 3: sync_c=4.475 +[Apr 27, 2026 - 21:02:45 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 21:02:47 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.8s +[Apr 27, 2026 - 21:02:50 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 4: sync_c=4.878 +[Apr 27, 2026 - 21:02:50 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 21:02:52 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.8s +[Apr 27, 2026 - 21:02:55 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 5: sync_c=6.130 +[Apr 27, 2026 - 21:02:55 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 21:02:57 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.8s +[Apr 27, 2026 - 21:03:00 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 6: sync_c=6.615 +[Apr 27, 2026 - 21:03:00 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 21:03:02 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.8s +[Apr 27, 2026 - 21:03:05 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 7: sync_c=5.141 +[Apr 27, 2026 - 21:03:05 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 21:03:07 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.8s +[Apr 27, 2026 - 21:03:10 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 8: sync_c=5.577 +[Apr 27, 2026 - 21:03:10 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 21:03:12 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.8s +[Apr 27, 2026 - 21:03:15 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 9: sync_c=3.315 +[Apr 27, 2026 - 21:03:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging val0 stats at iteration 500-------------------- +[Apr 27, 2026 - 21:03:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0000 iter count: 10.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[Apr 27, 2026 - 21:03:32 | INFO | fastgen.callbacks.wandb:on_validation_end:763 ] Logged 10 val videos at iteration 500 +[Apr 27, 2026 - 21:03:32 | INFO | fastgen.callbacks.wandb:on_validation_end:769 ] [SyncEval] val0 mean sync_c: 5.182 (n=10) +[Apr 27, 2026 - 21:04:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313534 43.372726 43.201725 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 71.476562 72.828125 69.820312 +util 98.250000 99.000000 96.000000 +[Apr 27, 2026 - 21:04:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 501 : avg iteration time 152.45 seconds +[Apr 27, 2026 - 21:04:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 501 : data loading time 0.00 +[Apr 27, 2026 - 21:04:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 501 : avg forward pass time 13.88 +[Apr 27, 2026 - 21:04:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 501 : backward pass time 11.90 +[Apr 27, 2026 - 21:04:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 501 : optimizer step time 0.97 +[Apr 27, 2026 - 21:04:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 501-------------------- +[Apr 27, 2026 - 21:04:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0220 iter count: 1.0 +[Apr 27, 2026 - 21:04:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0220 iter count: 1.0 +[Apr 27, 2026 - 21:04:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 21:04:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 21:05:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313434 43.372730 43.201736 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 71.520508 72.984375 69.839844 +util 98.750000 100.000000 97.000000 +[Apr 27, 2026 - 21:05:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 502 : avg iteration time 50.13 seconds +[Apr 27, 2026 - 21:05:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 502 : data loading time 0.00 +[Apr 27, 2026 - 21:05:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 502 : avg forward pass time 12.80 +[Apr 27, 2026 - 21:05:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 502 : backward pass time 11.55 +[Apr 27, 2026 - 21:05:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 502 : optimizer step time 0.99 +[Apr 27, 2026 - 21:05:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 502-------------------- +[Apr 27, 2026 - 21:05:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0182 iter count: 1.0 +[Apr 27, 2026 - 21:05:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0182 iter count: 1.0 +[Apr 27, 2026 - 21:05:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 21:05:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 21:06:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313465 43.372810 43.201759 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 71.476562 72.828125 69.898438 +util 97.000000 100.000000 91.000000 +[Apr 27, 2026 - 21:06:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 503 : avg iteration time 50.61 seconds +[Apr 27, 2026 - 21:06:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 503 : data loading time 0.00 +[Apr 27, 2026 - 21:06:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 503 : avg forward pass time 12.72 +[Apr 27, 2026 - 21:06:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 503 : backward pass time 11.61 +[Apr 27, 2026 - 21:06:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 503 : optimizer step time 0.99 +[Apr 27, 2026 - 21:06:05 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 503-------------------- +[Apr 27, 2026 - 21:06:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0193 iter count: 1.0 +[Apr 27, 2026 - 21:06:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0193 iter count: 1.0 +[Apr 27, 2026 - 21:06:05 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 21:06:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 21:07:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.312929 43.372791 43.201744 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 71.452148 72.847656 69.800781 +util 94.000000 100.000000 90.000000 +[Apr 27, 2026 - 21:07:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 504 : avg iteration time 54.76 seconds +[Apr 27, 2026 - 21:07:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 504 : data loading time 0.00 +[Apr 27, 2026 - 21:07:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 504 : avg forward pass time 14.41 +[Apr 27, 2026 - 21:07:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 504 : backward pass time 13.10 +[Apr 27, 2026 - 21:07:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 504 : optimizer step time 1.26 +[Apr 27, 2026 - 21:07:00 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 504-------------------- +[Apr 27, 2026 - 21:07:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0198 iter count: 1.0 +[Apr 27, 2026 - 21:07:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0198 iter count: 1.0 +[Apr 27, 2026 - 21:07:00 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 21:07:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 21:09:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.312852 43.373238 43.202301 +peak_gpu_mem_gb 129.581032 131.620712 123.567759 +peak_gpu_mem_reserved_gb 133.623535 135.638672 127.667969 +util 70.750000 77.000000 67.000000 +[Apr 27, 2026 - 21:09:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 505 : avg iteration time 175.63 seconds +[Apr 27, 2026 - 21:09:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 505 : data loading time 0.00 +[Apr 27, 2026 - 21:09:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 505 : avg forward pass time 79.32 +[Apr 27, 2026 - 21:09:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 505 : backward pass time 8.34 +[Apr 27, 2026 - 21:09:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 505 : optimizer step time 0.34 +[Apr 27, 2026 - 21:09:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 505-------------------- +[Apr 27, 2026 - 21:09:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 211187.3672 iter count: 1.0 +[Apr 27, 2026 - 21:09:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6506 iter count: 1.0 +[Apr 27, 2026 - 21:09:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 211187.3672iter count: 1.0 +[Apr 27, 2026 - 21:09:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 21:09:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.8086iter count: 1.0 +[Apr 27, 2026 - 21:09:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 2.8386 iter count: 1.0 +[Apr 27, 2026 - 21:09:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.5514 iter count: 1.0 +[Apr 27, 2026 - 21:09:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 272520.4375iter count: 1.0 +[Apr 27, 2026 - 21:09:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 292.1521iter count: 1.0 +[Apr 27, 2026 - 21:09:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 3623155.0000iter count: 1.0 +[Apr 27, 2026 - 21:09:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6506iter count: 1.0 +[Apr 27, 2026 - 21:09:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.9784 iter count: 1.0 +[Apr 27, 2026 - 21:09:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 2854.8313iter count: 1.0 +[Apr 27, 2026 - 21:09:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 2.8386 iter count: 1.0 +[Apr 27, 2026 - 21:09:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 292.1521iter count: 1.0 +[Apr 27, 2026 - 21:09:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.8295 iter count: 1.0 +[Apr 27, 2026 - 21:09:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 15663.3750iter count: 1.0 +[Apr 27, 2026 - 21:09:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 2.9114 iter count: 1.0 +[Apr 27, 2026 - 21:09:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 337.8904iter count: 1.0 +[Apr 27, 2026 - 21:09:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0244 iter count: 1.0 +[Apr 27, 2026 - 21:09:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 21:09:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 21:10:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.312855 43.373238 43.202305 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.051270 70.099609 70.015625 +util 98.750000 100.000000 97.000000 +[Apr 27, 2026 - 21:10:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 506 : avg iteration time 52.09 seconds +[Apr 27, 2026 - 21:10:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 506 : data loading time 0.00 +[Apr 27, 2026 - 21:10:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 506 : avg forward pass time 13.54 +[Apr 27, 2026 - 21:10:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 506 : backward pass time 11.59 +[Apr 27, 2026 - 21:10:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 506 : optimizer step time 0.97 +[Apr 27, 2026 - 21:10:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 506-------------------- +[Apr 27, 2026 - 21:10:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0225 iter count: 1.0 +[Apr 27, 2026 - 21:10:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0225 iter count: 1.0 +[Apr 27, 2026 - 21:10:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 21:10:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 21:11:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.312875 43.373299 43.202305 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.997070 70.035156 69.902344 +util 96.000000 99.000000 92.000000 +[Apr 27, 2026 - 21:11:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 507 : avg iteration time 50.46 seconds +[Apr 27, 2026 - 21:11:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 507 : data loading time 0.00 +[Apr 27, 2026 - 21:11:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 507 : avg forward pass time 12.59 +[Apr 27, 2026 - 21:11:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 507 : backward pass time 11.54 +[Apr 27, 2026 - 21:11:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 507 : optimizer step time 1.15 +[Apr 27, 2026 - 21:11:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 507-------------------- +[Apr 27, 2026 - 21:11:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0226 iter count: 1.0 +[Apr 27, 2026 - 21:11:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0226 iter count: 1.0 +[Apr 27, 2026 - 21:11:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 21:11:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 21:12:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.312878 43.373280 43.202335 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.016602 70.078125 69.917969 +util 95.500000 100.000000 91.000000 +[Apr 27, 2026 - 21:12:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 508 : avg iteration time 50.76 seconds +[Apr 27, 2026 - 21:12:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 508 : data loading time 0.00 +[Apr 27, 2026 - 21:12:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 508 : avg forward pass time 12.72 +[Apr 27, 2026 - 21:12:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 508 : backward pass time 12.73 +[Apr 27, 2026 - 21:12:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 508 : optimizer step time 1.03 +[Apr 27, 2026 - 21:12:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 508-------------------- +[Apr 27, 2026 - 21:12:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0189 iter count: 1.0 +[Apr 27, 2026 - 21:12:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0189 iter count: 1.0 +[Apr 27, 2026 - 21:12:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 21:12:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 21:13:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.312877 43.373299 43.202290 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.972656 70.035156 69.898438 +util 97.750000 100.000000 93.000000 +[Apr 27, 2026 - 21:13:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 509 : avg iteration time 51.93 seconds +[Apr 27, 2026 - 21:13:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 509 : data loading time 0.00 +[Apr 27, 2026 - 21:13:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 509 : avg forward pass time 13.90 +[Apr 27, 2026 - 21:13:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 509 : backward pass time 11.58 +[Apr 27, 2026 - 21:13:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 509 : optimizer step time 0.99 +[Apr 27, 2026 - 21:13:21 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 509-------------------- +[Apr 27, 2026 - 21:13:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0231 iter count: 1.0 +[Apr 27, 2026 - 21:13:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0231 iter count: 1.0 +[Apr 27, 2026 - 21:13:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 21:13:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 21:16:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.312956 43.373280 43.202503 +peak_gpu_mem_gb 129.581032 131.620712 123.567759 +peak_gpu_mem_reserved_gb 133.632812 135.658203 127.607422 +util 68.750000 75.000000 65.000000 +[Apr 27, 2026 - 21:16:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 510 : avg iteration time 173.93 seconds +[Apr 27, 2026 - 21:16:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 510 : data loading time 0.00 +[Apr 27, 2026 - 21:16:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 510 : avg forward pass time 78.43 +[Apr 27, 2026 - 21:16:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 510 : backward pass time 8.38 +[Apr 27, 2026 - 21:16:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 510 : optimizer step time 0.34 +[Apr 27, 2026 - 21:16:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 510-------------------- +[Apr 27, 2026 - 21:16:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 557581.1067 iter count: 1.0 +[Apr 27, 2026 - 21:16:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6249 iter count: 1.0 +[Apr 27, 2026 - 21:16:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 557581.1067iter count: 1.0 +[Apr 27, 2026 - 21:16:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 21:16:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.7875iter count: 1.0 +[Apr 27, 2026 - 21:16:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.1274 iter count: 1.0 +[Apr 27, 2026 - 21:16:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.1660 iter count: 1.0 +[Apr 27, 2026 - 21:16:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 828342.5000iter count: 1.0 +[Apr 27, 2026 - 21:16:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 9.5331 iter count: 1.0 +[Apr 27, 2026 - 21:16:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 12385098.0000iter count: 1.0 +[Apr 27, 2026 - 21:16:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6249iter count: 1.0 +[Apr 27, 2026 - 21:16:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.9713 iter count: 1.0 +[Apr 27, 2026 - 21:16:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 2814.6270iter count: 1.0 +[Apr 27, 2026 - 21:16:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 6.1844 iter count: 1.0 +[Apr 27, 2026 - 21:16:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 235362.7188iter count: 1.0 +[Apr 27, 2026 - 21:16:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.8104 iter count: 1.0 +[Apr 27, 2026 - 21:16:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 15075.9395iter count: 1.0 +[Apr 27, 2026 - 21:16:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.8032 iter count: 1.0 +[Apr 27, 2026 - 21:16:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 2011.2238iter count: 1.0 +[Apr 27, 2026 - 21:16:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0240 iter count: 1.0 +[Apr 27, 2026 - 21:16:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 21:16:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 21:17:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.312958 43.373306 43.202503 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.987793 70.197266 69.898438 +util 98.250000 100.000000 96.000000 +[Apr 27, 2026 - 21:17:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 511 : avg iteration time 55.22 seconds +[Apr 27, 2026 - 21:17:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 511 : data loading time 0.00 +[Apr 27, 2026 - 21:17:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 511 : avg forward pass time 14.93 +[Apr 27, 2026 - 21:17:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 511 : backward pass time 11.79 +[Apr 27, 2026 - 21:17:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 511 : optimizer step time 1.50 +[Apr 27, 2026 - 21:17:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 511-------------------- +[Apr 27, 2026 - 21:17:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0237 iter count: 1.0 +[Apr 27, 2026 - 21:17:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0237 iter count: 1.0 +[Apr 27, 2026 - 21:17:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 21:17:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 21:18:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.312974 43.373333 43.202538 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.948730 70.158203 69.800781 +util 97.500000 99.000000 93.000000 +[Apr 27, 2026 - 21:18:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 512 : avg iteration time 50.85 seconds +[Apr 27, 2026 - 21:18:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 512 : data loading time 0.00 +[Apr 27, 2026 - 21:18:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 512 : avg forward pass time 12.86 +[Apr 27, 2026 - 21:18:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 512 : backward pass time 12.54 +[Apr 27, 2026 - 21:18:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 512 : optimizer step time 1.01 +[Apr 27, 2026 - 21:18:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 512-------------------- +[Apr 27, 2026 - 21:18:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0243 iter count: 1.0 +[Apr 27, 2026 - 21:18:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0243 iter count: 1.0 +[Apr 27, 2026 - 21:18:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 21:18:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 21:18:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.312988 43.373360 43.202560 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.880371 70.001953 69.800781 +util 95.000000 99.000000 92.000000 +[Apr 27, 2026 - 21:18:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 513 : avg iteration time 47.50 seconds +[Apr 27, 2026 - 21:18:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 513 : data loading time 0.00 +[Apr 27, 2026 - 21:18:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 513 : avg forward pass time 11.67 +[Apr 27, 2026 - 21:18:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 513 : backward pass time 11.54 +[Apr 27, 2026 - 21:18:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 513 : optimizer step time 1.02 +[Apr 27, 2026 - 21:18:48 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 513-------------------- +[Apr 27, 2026 - 21:18:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0208 iter count: 1.0 +[Apr 27, 2026 - 21:18:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0208 iter count: 1.0 +[Apr 27, 2026 - 21:18:48 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 21:18:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 21:19:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313019 43.373470 43.202560 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.895020 69.982422 69.800781 +util 98.750000 100.000000 96.000000 +[Apr 27, 2026 - 21:19:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 514 : avg iteration time 48.13 seconds +[Apr 27, 2026 - 21:19:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 514 : data loading time 0.00 +[Apr 27, 2026 - 21:19:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 514 : avg forward pass time 11.55 +[Apr 27, 2026 - 21:19:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 514 : backward pass time 11.56 +[Apr 27, 2026 - 21:19:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 514 : optimizer step time 0.97 +[Apr 27, 2026 - 21:19:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 514-------------------- +[Apr 27, 2026 - 21:19:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0203 iter count: 1.0 +[Apr 27, 2026 - 21:19:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0203 iter count: 1.0 +[Apr 27, 2026 - 21:19:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 21:19:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 21:22:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.312640 43.373470 43.202560 +peak_gpu_mem_gb 129.581032 131.620712 123.567759 +peak_gpu_mem_reserved_gb 133.541016 135.560547 127.568359 +util 72.500000 76.000000 66.000000 +[Apr 27, 2026 - 21:22:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 515 : avg iteration time 176.60 seconds +[Apr 27, 2026 - 21:22:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 515 : data loading time 0.00 +[Apr 27, 2026 - 21:22:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 515 : avg forward pass time 79.76 +[Apr 27, 2026 - 21:22:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 515 : backward pass time 8.38 +[Apr 27, 2026 - 21:22:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 515 : optimizer step time 0.34 +[Apr 27, 2026 - 21:22:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 515-------------------- +[Apr 27, 2026 - 21:22:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 288609.6313 iter count: 1.0 +[Apr 27, 2026 - 21:22:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5616 iter count: 1.0 +[Apr 27, 2026 - 21:22:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 288609.6313iter count: 1.0 +[Apr 27, 2026 - 21:22:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 21:22:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.2635iter count: 1.0 +[Apr 27, 2026 - 21:22:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.7707 iter count: 1.0 +[Apr 27, 2026 - 21:22:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.9580 iter count: 1.0 +[Apr 27, 2026 - 21:22:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 356991.0000iter count: 1.0 +[Apr 27, 2026 - 21:22:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 4.6710 iter count: 1.0 +[Apr 27, 2026 - 21:22:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 8169944.5000iter count: 1.0 +[Apr 27, 2026 - 21:22:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5616iter count: 1.0 +[Apr 27, 2026 - 21:22:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 6.7596 iter count: 1.0 +[Apr 27, 2026 - 21:22:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 743559.5625iter count: 1.0 +[Apr 27, 2026 - 21:22:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 6.0471 iter count: 1.0 +[Apr 27, 2026 - 21:22:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 178833.1875iter count: 1.0 +[Apr 27, 2026 - 21:22:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.4107 iter count: 1.0 +[Apr 27, 2026 - 21:22:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 6777.6675iter count: 1.0 +[Apr 27, 2026 - 21:22:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.7282 iter count: 1.0 +[Apr 27, 2026 - 21:22:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 1730.8461iter count: 1.0 +[Apr 27, 2026 - 21:22:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0228 iter count: 1.0 +[Apr 27, 2026 - 21:22:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 21:22:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 21:23:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.312640 43.373455 43.202576 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.977539 70.000000 69.957031 +util 96.750000 100.000000 93.000000 +[Apr 27, 2026 - 21:23:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 516 : avg iteration time 55.08 seconds +[Apr 27, 2026 - 21:23:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 516 : data loading time 0.00 +[Apr 27, 2026 - 21:23:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 516 : avg forward pass time 14.85 +[Apr 27, 2026 - 21:23:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 516 : backward pass time 11.64 +[Apr 27, 2026 - 21:23:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 516 : optimizer step time 1.39 +[Apr 27, 2026 - 21:23:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 516-------------------- +[Apr 27, 2026 - 21:23:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0276 iter count: 1.0 +[Apr 27, 2026 - 21:23:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0276 iter count: 1.0 +[Apr 27, 2026 - 21:23:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 21:23:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 21:24:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.312913 43.373501 43.202606 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.914062 70.019531 69.859375 +util 92.500000 99.000000 89.000000 +[Apr 27, 2026 - 21:24:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 517 : avg iteration time 48.18 seconds +[Apr 27, 2026 - 21:24:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 517 : data loading time 0.00 +[Apr 27, 2026 - 21:24:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 517 : avg forward pass time 11.17 +[Apr 27, 2026 - 21:24:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 517 : backward pass time 13.01 +[Apr 27, 2026 - 21:24:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 517 : optimizer step time 1.20 +[Apr 27, 2026 - 21:24:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 517-------------------- +[Apr 27, 2026 - 21:24:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0218 iter count: 1.0 +[Apr 27, 2026 - 21:24:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0218 iter count: 1.0 +[Apr 27, 2026 - 21:24:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 21:24:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 21:25:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.312982 43.373478 43.202599 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.972656 70.015625 69.917969 +util 98.500000 99.000000 97.000000 +[Apr 27, 2026 - 21:25:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 518 : avg iteration time 47.67 seconds +[Apr 27, 2026 - 21:25:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 518 : data loading time 0.00 +[Apr 27, 2026 - 21:25:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 518 : avg forward pass time 11.40 +[Apr 27, 2026 - 21:25:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 518 : backward pass time 12.29 +[Apr 27, 2026 - 21:25:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 518 : optimizer step time 1.01 +[Apr 27, 2026 - 21:25:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 518-------------------- +[Apr 27, 2026 - 21:25:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0217 iter count: 1.0 +[Apr 27, 2026 - 21:25:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0217 iter count: 1.0 +[Apr 27, 2026 - 21:25:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 21:25:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 21:25:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.312982 43.373501 43.202599 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 71.408203 72.886719 69.917969 +util 98.000000 99.000000 95.000000 +[Apr 27, 2026 - 21:25:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 519 : avg iteration time 49.66 seconds +[Apr 27, 2026 - 21:25:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 519 : data loading time 0.00 +[Apr 27, 2026 - 21:25:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 519 : avg forward pass time 12.70 +[Apr 27, 2026 - 21:25:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 519 : backward pass time 11.57 +[Apr 27, 2026 - 21:25:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 519 : optimizer step time 0.99 +[Apr 27, 2026 - 21:25:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 519-------------------- +[Apr 27, 2026 - 21:25:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0196 iter count: 1.0 +[Apr 27, 2026 - 21:25:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0196 iter count: 1.0 +[Apr 27, 2026 - 21:25:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 21:25:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 21:28:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.312685 43.373489 43.202629 +peak_gpu_mem_gb 129.581032 131.620712 123.567759 +peak_gpu_mem_reserved_gb 133.551270 135.677734 127.609375 +util 69.000000 74.000000 64.000000 +[Apr 27, 2026 - 21:28:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 520 : avg iteration time 181.70 seconds +[Apr 27, 2026 - 21:28:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 520 : data loading time 0.00 +[Apr 27, 2026 - 21:28:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 520 : avg forward pass time 82.34 +[Apr 27, 2026 - 21:28:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 520 : backward pass time 8.38 +[Apr 27, 2026 - 21:28:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 520 : optimizer step time 0.34 +[Apr 27, 2026 - 21:28:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 520-------------------- +[Apr 27, 2026 - 21:28:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 2462427.1455 iter count: 1.0 +[Apr 27, 2026 - 21:28:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5280 iter count: 1.0 +[Apr 27, 2026 - 21:28:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 2462427.1455iter count: 1.0 +[Apr 27, 2026 - 21:28:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 21:28:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.5729iter count: 1.0 +[Apr 27, 2026 - 21:28:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.7617 iter count: 1.0 +[Apr 27, 2026 - 21:28:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 9.0569 iter count: 1.0 +[Apr 27, 2026 - 21:28:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 2795893.5000iter count: 1.0 +[Apr 27, 2026 - 21:28:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 4.5882 iter count: 1.0 +[Apr 27, 2026 - 21:28:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 73576928.0000iter count: 1.0 +[Apr 27, 2026 - 21:28:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5280iter count: 1.0 +[Apr 27, 2026 - 21:28:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 1.2257 iter count: 1.0 +[Apr 27, 2026 - 21:28:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 11.6055 iter count: 1.0 +[Apr 27, 2026 - 21:28:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.9420 iter count: 1.0 +[Apr 27, 2026 - 21:28:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 2654.3853iter count: 1.0 +[Apr 27, 2026 - 21:28:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.4826 iter count: 1.0 +[Apr 27, 2026 - 21:28:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 7825.9653iter count: 1.0 +[Apr 27, 2026 - 21:28:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.5506 iter count: 1.0 +[Apr 27, 2026 - 21:28:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 489556.1562iter count: 1.0 +[Apr 27, 2026 - 21:28:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0245 iter count: 1.0 +[Apr 27, 2026 - 21:28:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 21:28:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 21:29:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.312765 43.373600 43.202637 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.914062 70.054688 69.843750 +util 97.250000 99.000000 92.000000 +[Apr 27, 2026 - 21:29:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 521 : avg iteration time 52.27 seconds +[Apr 27, 2026 - 21:29:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 521 : data loading time 0.00 +[Apr 27, 2026 - 21:29:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 521 : avg forward pass time 13.75 +[Apr 27, 2026 - 21:29:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 521 : backward pass time 12.21 +[Apr 27, 2026 - 21:29:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 521 : optimizer step time 0.99 +[Apr 27, 2026 - 21:29:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 521-------------------- +[Apr 27, 2026 - 21:29:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0219 iter count: 1.0 +[Apr 27, 2026 - 21:29:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0219 iter count: 1.0 +[Apr 27, 2026 - 21:29:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 21:29:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 21:30:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313201 43.373589 43.202637 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.967773 70.054688 69.863281 +util 97.500000 99.000000 96.000000 +[Apr 27, 2026 - 21:30:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 522 : avg iteration time 52.36 seconds +[Apr 27, 2026 - 21:30:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 522 : data loading time 0.00 +[Apr 27, 2026 - 21:30:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 522 : avg forward pass time 14.06 +[Apr 27, 2026 - 21:30:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 522 : backward pass time 11.55 +[Apr 27, 2026 - 21:30:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 522 : optimizer step time 0.98 +[Apr 27, 2026 - 21:30:40 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 522-------------------- +[Apr 27, 2026 - 21:30:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0243 iter count: 1.0 +[Apr 27, 2026 - 21:30:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0243 iter count: 1.0 +[Apr 27, 2026 - 21:30:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 21:30:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 21:31:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313221 43.373611 43.202637 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.031250 70.230469 69.863281 +util 95.750000 100.000000 91.000000 +[Apr 27, 2026 - 21:31:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 523 : avg iteration time 47.93 seconds +[Apr 27, 2026 - 21:31:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 523 : data loading time 0.00 +[Apr 27, 2026 - 21:31:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 523 : avg forward pass time 11.41 +[Apr 27, 2026 - 21:31:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 523 : backward pass time 11.54 +[Apr 27, 2026 - 21:31:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 523 : optimizer step time 1.01 +[Apr 27, 2026 - 21:31:28 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 523-------------------- +[Apr 27, 2026 - 21:31:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0244 iter count: 1.0 +[Apr 27, 2026 - 21:31:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0244 iter count: 1.0 +[Apr 27, 2026 - 21:31:28 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 21:31:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 21:32:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.312970 43.373589 43.202644 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.948242 70.054688 69.863281 +util 94.000000 99.000000 92.000000 +[Apr 27, 2026 - 21:32:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 524 : avg iteration time 54.61 seconds +[Apr 27, 2026 - 21:32:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 524 : data loading time 0.00 +[Apr 27, 2026 - 21:32:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 524 : avg forward pass time 14.43 +[Apr 27, 2026 - 21:32:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 524 : backward pass time 13.01 +[Apr 27, 2026 - 21:32:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 524 : optimizer step time 1.16 +[Apr 27, 2026 - 21:32:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 524-------------------- +[Apr 27, 2026 - 21:32:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0251 iter count: 1.0 +[Apr 27, 2026 - 21:32:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0251 iter count: 1.0 +[Apr 27, 2026 - 21:32:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 21:32:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 21:35:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.312880 43.373608 43.202637 +peak_gpu_mem_gb 129.581032 131.620712 123.567759 +peak_gpu_mem_reserved_gb 134.078613 135.677734 129.523438 +util 70.500000 77.000000 67.000000 +[Apr 27, 2026 - 21:35:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 525 : avg iteration time 171.28 seconds +[Apr 27, 2026 - 21:35:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 525 : data loading time 0.00 +[Apr 27, 2026 - 21:35:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 525 : avg forward pass time 77.13 +[Apr 27, 2026 - 21:35:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 525 : backward pass time 8.34 +[Apr 27, 2026 - 21:35:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 525 : optimizer step time 0.34 +[Apr 27, 2026 - 21:35:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 525-------------------- +[Apr 27, 2026 - 21:35:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 49066.8810 iter count: 1.0 +[Apr 27, 2026 - 21:35:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6052 iter count: 1.0 +[Apr 27, 2026 - 21:35:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 49066.8810iter count: 1.0 +[Apr 27, 2026 - 21:35:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 21:35:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 3.8916iter count: 1.0 +[Apr 27, 2026 - 21:35:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.7577 iter count: 1.0 +[Apr 27, 2026 - 21:35:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.0089 iter count: 1.0 +[Apr 27, 2026 - 21:35:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 91297.2891iter count: 1.0 +[Apr 27, 2026 - 21:35:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 4.5516 iter count: 1.0 +[Apr 27, 2026 - 21:35:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 1224095.2500iter count: 1.0 +[Apr 27, 2026 - 21:35:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6052iter count: 1.0 +[Apr 27, 2026 - 21:35:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 1.0379 iter count: 1.0 +[Apr 27, 2026 - 21:35:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 7.9717 iter count: 1.0 +[Apr 27, 2026 - 21:35:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.5471 iter count: 1.0 +[Apr 27, 2026 - 21:35:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 65784.0312iter count: 1.0 +[Apr 27, 2026 - 21:35:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 7.0089 iter count: 1.0 +[Apr 27, 2026 - 21:35:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 1224095.2500iter count: 1.0 +[Apr 27, 2026 - 21:35:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.3364 iter count: 1.0 +[Apr 27, 2026 - 21:35:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 790.6409iter count: 1.0 +[Apr 27, 2026 - 21:35:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0227 iter count: 1.0 +[Apr 27, 2026 - 21:35:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 21:35:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 21:36:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.312948 43.373608 43.202660 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.587891 72.417969 69.902344 +util 95.500000 99.000000 93.000000 +[Apr 27, 2026 - 21:36:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 526 : avg iteration time 50.36 seconds +[Apr 27, 2026 - 21:36:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 526 : data loading time 0.00 +[Apr 27, 2026 - 21:36:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 526 : avg forward pass time 12.68 +[Apr 27, 2026 - 21:36:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 526 : backward pass time 11.68 +[Apr 27, 2026 - 21:36:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 526 : optimizer step time 1.00 +[Apr 27, 2026 - 21:36:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 526-------------------- +[Apr 27, 2026 - 21:36:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0196 iter count: 1.0 +[Apr 27, 2026 - 21:36:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0196 iter count: 1.0 +[Apr 27, 2026 - 21:36:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 21:36:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 21:36:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.312954 43.373623 43.202667 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.602539 72.417969 69.996094 +util 97.750000 99.000000 94.000000 +[Apr 27, 2026 - 21:36:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 527 : avg iteration time 52.90 seconds +[Apr 27, 2026 - 21:36:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 527 : data loading time 0.00 +[Apr 27, 2026 - 21:36:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 527 : avg forward pass time 14.10 +[Apr 27, 2026 - 21:36:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 527 : backward pass time 11.57 +[Apr 27, 2026 - 21:36:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 527 : optimizer step time 0.98 +[Apr 27, 2026 - 21:36:57 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 527-------------------- +[Apr 27, 2026 - 21:36:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0255 iter count: 1.0 +[Apr 27, 2026 - 21:36:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0255 iter count: 1.0 +[Apr 27, 2026 - 21:36:57 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 21:36:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 21:37:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.312960 43.373623 43.202690 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.656250 72.457031 70.015625 +util 96.750000 99.000000 91.000000 +[Apr 27, 2026 - 21:37:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 528 : avg iteration time 47.76 seconds +[Apr 27, 2026 - 21:37:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 528 : data loading time 0.00 +[Apr 27, 2026 - 21:37:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 528 : avg forward pass time 11.27 +[Apr 27, 2026 - 21:37:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 528 : backward pass time 11.54 +[Apr 27, 2026 - 21:37:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 528 : optimizer step time 0.98 +[Apr 27, 2026 - 21:37:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 528-------------------- +[Apr 27, 2026 - 21:37:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0204 iter count: 1.0 +[Apr 27, 2026 - 21:37:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0204 iter count: 1.0 +[Apr 27, 2026 - 21:37:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 21:37:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 21:38:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313023 43.373623 43.202686 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.656250 72.457031 70.015625 +util 97.500000 100.000000 92.000000 +[Apr 27, 2026 - 21:38:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 529 : avg iteration time 54.61 seconds +[Apr 27, 2026 - 21:38:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 529 : data loading time 0.00 +[Apr 27, 2026 - 21:38:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 529 : avg forward pass time 14.33 +[Apr 27, 2026 - 21:38:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 529 : backward pass time 13.12 +[Apr 27, 2026 - 21:38:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 529 : optimizer step time 1.29 +[Apr 27, 2026 - 21:38:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 529-------------------- +[Apr 27, 2026 - 21:38:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0228 iter count: 1.0 +[Apr 27, 2026 - 21:38:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0228 iter count: 1.0 +[Apr 27, 2026 - 21:38:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 21:38:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 21:41:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.312875 43.373623 43.202671 +peak_gpu_mem_gb 129.581032 131.620712 123.567759 +peak_gpu_mem_reserved_gb 133.525879 135.658203 127.488281 +util 75.250000 78.000000 69.000000 +[Apr 27, 2026 - 21:41:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 530 : avg iteration time 170.47 seconds +[Apr 27, 2026 - 21:41:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 530 : data loading time 0.01 +[Apr 27, 2026 - 21:41:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 530 : avg forward pass time 76.69 +[Apr 27, 2026 - 21:41:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 530 : backward pass time 8.38 +[Apr 27, 2026 - 21:41:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 530 : optimizer step time 0.34 +[Apr 27, 2026 - 21:41:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 530-------------------- +[Apr 27, 2026 - 21:41:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 245772.6372 iter count: 1.0 +[Apr 27, 2026 - 21:41:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6600 iter count: 1.0 +[Apr 27, 2026 - 21:41:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 245772.6372iter count: 1.0 +[Apr 27, 2026 - 21:41:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 21:41:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.6816iter count: 1.0 +[Apr 27, 2026 - 21:41:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 2.4737 iter count: 1.0 +[Apr 27, 2026 - 21:41:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.7889 iter count: 1.0 +[Apr 27, 2026 - 21:41:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 381411.6562iter count: 1.0 +[Apr 27, 2026 - 21:41:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 140.8072iter count: 1.0 +[Apr 27, 2026 - 21:41:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 5825608.5000iter count: 1.0 +[Apr 27, 2026 - 21:41:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6600iter count: 1.0 +[Apr 27, 2026 - 21:41:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 2.5018 iter count: 1.0 +[Apr 27, 2026 - 21:41:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 148.9486iter count: 1.0 +[Apr 27, 2026 - 21:41:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.7781 iter count: 1.0 +[Apr 27, 2026 - 21:41:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 14131.6924iter count: 1.0 +[Apr 27, 2026 - 21:41:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.8102 iter count: 1.0 +[Apr 27, 2026 - 21:41:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 15070.5342iter count: 1.0 +[Apr 27, 2026 - 21:41:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.5471 iter count: 1.0 +[Apr 27, 2026 - 21:41:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 65785.6641iter count: 1.0 +[Apr 27, 2026 - 21:41:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0192 iter count: 1.0 +[Apr 27, 2026 - 21:41:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 21:41:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 21:42:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313076 43.373623 43.202728 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.958008 70.078125 69.878906 +util 94.500000 99.000000 92.000000 +[Apr 27, 2026 - 21:42:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 531 : avg iteration time 54.84 seconds +[Apr 27, 2026 - 21:42:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 531 : data loading time 0.00 +[Apr 27, 2026 - 21:42:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 531 : avg forward pass time 14.61 +[Apr 27, 2026 - 21:42:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 531 : backward pass time 11.54 +[Apr 27, 2026 - 21:42:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 531 : optimizer step time 0.99 +[Apr 27, 2026 - 21:42:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 531-------------------- +[Apr 27, 2026 - 21:42:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0266 iter count: 1.0 +[Apr 27, 2026 - 21:42:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0266 iter count: 1.0 +[Apr 27, 2026 - 21:42:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 21:42:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 21:43:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313079 43.373623 43.202728 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.962891 70.117188 69.820312 +util 96.750000 99.000000 91.000000 +[Apr 27, 2026 - 21:43:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 532 : avg iteration time 54.47 seconds +[Apr 27, 2026 - 21:43:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 532 : data loading time 0.00 +[Apr 27, 2026 - 21:43:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 532 : avg forward pass time 14.29 +[Apr 27, 2026 - 21:43:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 532 : backward pass time 12.84 +[Apr 27, 2026 - 21:43:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 532 : optimizer step time 1.52 +[Apr 27, 2026 - 21:43:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 532-------------------- +[Apr 27, 2026 - 21:43:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0255 iter count: 1.0 +[Apr 27, 2026 - 21:43:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0255 iter count: 1.0 +[Apr 27, 2026 - 21:43:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 21:43:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 21:44:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313071 43.373638 43.202682 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.060547 70.117188 70.015625 +util 97.000000 99.000000 91.000000 +[Apr 27, 2026 - 21:44:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 533 : avg iteration time 50.41 seconds +[Apr 27, 2026 - 21:44:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 533 : data loading time 0.00 +[Apr 27, 2026 - 21:44:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 533 : avg forward pass time 12.75 +[Apr 27, 2026 - 21:44:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 533 : backward pass time 12.35 +[Apr 27, 2026 - 21:44:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 533 : optimizer step time 1.00 +[Apr 27, 2026 - 21:44:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 533-------------------- +[Apr 27, 2026 - 21:44:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0248 iter count: 1.0 +[Apr 27, 2026 - 21:44:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0248 iter count: 1.0 +[Apr 27, 2026 - 21:44:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 21:44:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 21:44:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313071 43.373623 43.202686 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.050781 70.117188 69.976562 +util 98.750000 100.000000 97.000000 +[Apr 27, 2026 - 21:44:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 534 : avg iteration time 49.26 seconds +[Apr 27, 2026 - 21:44:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 534 : data loading time 0.00 +[Apr 27, 2026 - 21:44:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 534 : avg forward pass time 12.54 +[Apr 27, 2026 - 21:44:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 534 : backward pass time 11.60 +[Apr 27, 2026 - 21:44:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 534 : optimizer step time 0.98 +[Apr 27, 2026 - 21:44:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 534-------------------- +[Apr 27, 2026 - 21:44:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0243 iter count: 1.0 +[Apr 27, 2026 - 21:44:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0243 iter count: 1.0 +[Apr 27, 2026 - 21:44:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 21:44:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 21:47:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.312782 43.373627 43.202682 +peak_gpu_mem_gb 129.581032 131.620712 123.567759 +peak_gpu_mem_reserved_gb 133.492188 135.580078 127.470703 +util 70.500000 77.000000 67.000000 +[Apr 27, 2026 - 21:47:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 535 : avg iteration time 173.12 seconds +[Apr 27, 2026 - 21:47:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 535 : data loading time 0.00 +[Apr 27, 2026 - 21:47:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 535 : avg forward pass time 78.06 +[Apr 27, 2026 - 21:47:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 535 : backward pass time 8.39 +[Apr 27, 2026 - 21:47:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 535 : optimizer step time 0.34 +[Apr 27, 2026 - 21:47:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 535-------------------- +[Apr 27, 2026 - 21:47:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 1276239.6484 iter count: 1.0 +[Apr 27, 2026 - 21:47:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6984 iter count: 1.0 +[Apr 27, 2026 - 21:47:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 1276239.6484iter count: 1.0 +[Apr 27, 2026 - 21:47:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 21:47:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.4798iter count: 1.0 +[Apr 27, 2026 - 21:47:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.8111 iter count: 1.0 +[Apr 27, 2026 - 21:47:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.7518 iter count: 1.0 +[Apr 27, 2026 - 21:47:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 1618391.5000iter count: 1.0 +[Apr 27, 2026 - 21:47:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 5.0647 iter count: 1.0 +[Apr 27, 2026 - 21:47:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 39964488.0000iter count: 1.0 +[Apr 27, 2026 - 21:47:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6984iter count: 1.0 +[Apr 27, 2026 - 21:47:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 7.7757 iter count: 1.0 +[Apr 27, 2026 - 21:47:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 5674444.0000iter count: 1.0 +[Apr 27, 2026 - 21:47:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 2.3057 iter count: 1.0 +[Apr 27, 2026 - 21:47:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 100.6271iter count: 1.0 +[Apr 27, 2026 - 21:47:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.5541 iter count: 1.0 +[Apr 27, 2026 - 21:47:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 9029.8037iter count: 1.0 +[Apr 27, 2026 - 21:47:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 7.4124 iter count: 1.0 +[Apr 27, 2026 - 21:47:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 2743848.5000iter count: 1.0 +[Apr 27, 2026 - 21:47:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0221 iter count: 1.0 +[Apr 27, 2026 - 21:47:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 21:47:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 21:48:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.312784 43.373627 43.202682 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.963379 70.015625 69.937500 +util 98.750000 100.000000 97.000000 +[Apr 27, 2026 - 21:48:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 536 : avg iteration time 51.81 seconds +[Apr 27, 2026 - 21:48:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 536 : data loading time 0.00 +[Apr 27, 2026 - 21:48:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 536 : avg forward pass time 13.46 +[Apr 27, 2026 - 21:48:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 536 : backward pass time 11.58 +[Apr 27, 2026 - 21:48:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 536 : optimizer step time 1.28 +[Apr 27, 2026 - 21:48:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 536-------------------- +[Apr 27, 2026 - 21:48:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0230 iter count: 1.0 +[Apr 27, 2026 - 21:48:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0230 iter count: 1.0 +[Apr 27, 2026 - 21:48:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 21:48:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 21:49:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313111 43.373650 43.202686 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.948730 69.976562 69.917969 +util 95.500000 99.000000 92.000000 +[Apr 27, 2026 - 21:49:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 537 : avg iteration time 52.92 seconds +[Apr 27, 2026 - 21:49:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 537 : data loading time 0.00 +[Apr 27, 2026 - 21:49:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 537 : avg forward pass time 14.28 +[Apr 27, 2026 - 21:49:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 537 : backward pass time 11.79 +[Apr 27, 2026 - 21:49:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 537 : optimizer step time 0.98 +[Apr 27, 2026 - 21:49:36 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 537-------------------- +[Apr 27, 2026 - 21:49:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0276 iter count: 1.0 +[Apr 27, 2026 - 21:49:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0276 iter count: 1.0 +[Apr 27, 2026 - 21:49:36 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 21:49:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 21:50:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313174 43.373634 43.202713 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.968262 70.001953 69.917969 +util 95.250000 99.000000 91.000000 +[Apr 27, 2026 - 21:50:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 538 : avg iteration time 49.59 seconds +[Apr 27, 2026 - 21:50:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 538 : data loading time 0.00 +[Apr 27, 2026 - 21:50:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 538 : avg forward pass time 12.67 +[Apr 27, 2026 - 21:50:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 538 : backward pass time 11.55 +[Apr 27, 2026 - 21:50:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 538 : optimizer step time 0.98 +[Apr 27, 2026 - 21:50:26 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 538-------------------- +[Apr 27, 2026 - 21:50:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0280 iter count: 1.0 +[Apr 27, 2026 - 21:50:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0280 iter count: 1.0 +[Apr 27, 2026 - 21:50:26 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 21:50:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 21:51:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313315 43.373684 43.202702 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.943848 69.976562 69.898438 +util 97.500000 100.000000 91.000000 +[Apr 27, 2026 - 21:51:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 539 : avg iteration time 53.80 seconds +[Apr 27, 2026 - 21:51:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 539 : data loading time 0.00 +[Apr 27, 2026 - 21:51:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 539 : avg forward pass time 14.20 +[Apr 27, 2026 - 21:51:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 539 : backward pass time 12.11 +[Apr 27, 2026 - 21:51:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 539 : optimizer step time 1.56 +[Apr 27, 2026 - 21:51:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 539-------------------- +[Apr 27, 2026 - 21:51:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0255 iter count: 1.0 +[Apr 27, 2026 - 21:51:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0255 iter count: 1.0 +[Apr 27, 2026 - 21:51:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 21:51:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 21:54:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.312940 43.373680 43.202713 +peak_gpu_mem_gb 129.581032 131.620712 123.567759 +peak_gpu_mem_reserved_gb 133.800293 135.619141 128.429688 +util 66.000000 72.000000 63.000000 +[Apr 27, 2026 - 21:54:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 540 : avg iteration time 175.03 seconds +[Apr 27, 2026 - 21:54:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 540 : data loading time 0.00 +[Apr 27, 2026 - 21:54:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 540 : avg forward pass time 78.24 +[Apr 27, 2026 - 21:54:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 540 : backward pass time 8.41 +[Apr 27, 2026 - 21:54:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 540 : optimizer step time 0.34 +[Apr 27, 2026 - 21:54:15 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 540-------------------- +[Apr 27, 2026 - 21:54:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 122997.1426 iter count: 1.0 +[Apr 27, 2026 - 21:54:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6508 iter count: 1.0 +[Apr 27, 2026 - 21:54:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 122997.1426iter count: 1.0 +[Apr 27, 2026 - 21:54:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 21:54:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.2622iter count: 1.0 +[Apr 27, 2026 - 21:54:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.0830 iter count: 1.0 +[Apr 27, 2026 - 21:54:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.2681 iter count: 1.0 +[Apr 27, 2026 - 21:54:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 168378.2500iter count: 1.0 +[Apr 27, 2026 - 21:54:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 8.7233 iter count: 1.0 +[Apr 27, 2026 - 21:54:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 2055837.5000iter count: 1.0 +[Apr 27, 2026 - 21:54:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6508iter count: 1.0 +[Apr 27, 2026 - 21:54:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 5.0790 iter count: 1.0 +[Apr 27, 2026 - 21:54:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 25794.4141iter count: 1.0 +[Apr 27, 2026 - 21:54:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.9438 iter count: 1.0 +[Apr 27, 2026 - 21:54:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 145439.7188iter count: 1.0 +[Apr 27, 2026 - 21:54:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.5907 iter count: 1.0 +[Apr 27, 2026 - 21:54:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 1314.7881iter count: 1.0 +[Apr 27, 2026 - 21:54:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.6277 iter count: 1.0 +[Apr 27, 2026 - 21:54:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 10461.3389iter count: 1.0 +[Apr 27, 2026 - 21:54:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0282 iter count: 1.0 +[Apr 27, 2026 - 21:54:15 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 21:54:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 21:55:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.312958 43.373665 43.202824 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.938965 70.021484 69.898438 +util 97.000000 99.000000 95.000000 +[Apr 27, 2026 - 21:55:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 541 : avg iteration time 54.04 seconds +[Apr 27, 2026 - 21:55:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 541 : data loading time 0.00 +[Apr 27, 2026 - 21:55:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 541 : avg forward pass time 14.75 +[Apr 27, 2026 - 21:55:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 541 : backward pass time 11.57 +[Apr 27, 2026 - 21:55:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 541 : optimizer step time 0.98 +[Apr 27, 2026 - 21:55:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 541-------------------- +[Apr 27, 2026 - 21:55:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0254 iter count: 1.0 +[Apr 27, 2026 - 21:55:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0254 iter count: 1.0 +[Apr 27, 2026 - 21:55:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 21:55:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 21:55:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313328 43.373665 43.202801 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.982910 70.021484 69.878906 +util 96.750000 99.000000 94.000000 +[Apr 27, 2026 - 21:55:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 542 : avg iteration time 50.58 seconds +[Apr 27, 2026 - 21:55:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 542 : data loading time 0.00 +[Apr 27, 2026 - 21:55:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 542 : avg forward pass time 12.61 +[Apr 27, 2026 - 21:55:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 542 : backward pass time 11.58 +[Apr 27, 2026 - 21:55:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 542 : optimizer step time 1.27 +[Apr 27, 2026 - 21:55:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 542-------------------- +[Apr 27, 2026 - 21:55:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0251 iter count: 1.0 +[Apr 27, 2026 - 21:55:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0251 iter count: 1.0 +[Apr 27, 2026 - 21:55:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 21:55:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 21:56:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313363 43.373775 43.202808 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.978027 70.021484 69.859375 +util 95.750000 99.000000 94.000000 +[Apr 27, 2026 - 21:56:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 543 : avg iteration time 50.60 seconds +[Apr 27, 2026 - 21:56:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 543 : data loading time 0.00 +[Apr 27, 2026 - 21:56:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 543 : avg forward pass time 12.66 +[Apr 27, 2026 - 21:56:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 543 : backward pass time 12.67 +[Apr 27, 2026 - 21:56:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 543 : optimizer step time 1.04 +[Apr 27, 2026 - 21:56:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 543-------------------- +[Apr 27, 2026 - 21:56:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0269 iter count: 1.0 +[Apr 27, 2026 - 21:56:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0269 iter count: 1.0 +[Apr 27, 2026 - 21:56:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 21:56:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 21:57:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313284 43.373775 43.202831 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.992676 70.021484 69.917969 +util 98.750000 100.000000 97.000000 +[Apr 27, 2026 - 21:57:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 544 : avg iteration time 51.73 seconds +[Apr 27, 2026 - 21:57:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 544 : data loading time 0.00 +[Apr 27, 2026 - 21:57:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 544 : avg forward pass time 13.81 +[Apr 27, 2026 - 21:57:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 544 : backward pass time 11.55 +[Apr 27, 2026 - 21:57:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 544 : optimizer step time 0.99 +[Apr 27, 2026 - 21:57:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 544-------------------- +[Apr 27, 2026 - 21:57:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0270 iter count: 1.0 +[Apr 27, 2026 - 21:57:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0270 iter count: 1.0 +[Apr 27, 2026 - 21:57:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 21:57:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 22:00:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313046 43.373779 43.202789 +peak_gpu_mem_gb 129.581032 131.620712 123.567759 +peak_gpu_mem_reserved_gb 134.175293 135.613281 129.933594 +util 67.250000 74.000000 64.000000 +[Apr 27, 2026 - 22:00:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 545 : avg iteration time 175.28 seconds +[Apr 27, 2026 - 22:00:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 545 : data loading time 0.00 +[Apr 27, 2026 - 22:00:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 545 : avg forward pass time 79.13 +[Apr 27, 2026 - 22:00:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 545 : backward pass time 8.35 +[Apr 27, 2026 - 22:00:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 545 : optimizer step time 0.35 +[Apr 27, 2026 - 22:00:37 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 545-------------------- +[Apr 27, 2026 - 22:00:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 279555.3145 iter count: 1.0 +[Apr 27, 2026 - 22:00:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6570 iter count: 1.0 +[Apr 27, 2026 - 22:00:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 279555.3145iter count: 1.0 +[Apr 27, 2026 - 22:00:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:00:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.7291iter count: 1.0 +[Apr 27, 2026 - 22:00:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.6505 iter count: 1.0 +[Apr 27, 2026 - 22:00:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.7729 iter count: 1.0 +[Apr 27, 2026 - 22:00:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 362202.1875iter count: 1.0 +[Apr 27, 2026 - 22:00:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 27.1418iter count: 1.0 +[Apr 27, 2026 - 22:00:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 5642067.5000iter count: 1.0 +[Apr 27, 2026 - 22:00:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6570iter count: 1.0 +[Apr 27, 2026 - 22:00:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 6.2084 iter count: 1.0 +[Apr 27, 2026 - 22:00:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 246898.5625iter count: 1.0 +[Apr 27, 2026 - 22:00:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.1996 iter count: 1.0 +[Apr 27, 2026 - 22:00:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 32835.3672iter count: 1.0 +[Apr 27, 2026 - 22:00:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.0165 iter count: 1.0 +[Apr 27, 2026 - 22:00:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 416.9819iter count: 1.0 +[Apr 27, 2026 - 22:00:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 7.7729 iter count: 1.0 +[Apr 27, 2026 - 22:00:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 5642067.5000iter count: 1.0 +[Apr 27, 2026 - 22:00:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0242 iter count: 1.0 +[Apr 27, 2026 - 22:00:37 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:00:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 22:01:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313047 43.373779 43.202824 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.909180 69.980469 69.859375 +util 95.250000 100.000000 93.000000 +[Apr 27, 2026 - 22:01:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 546 : avg iteration time 53.18 seconds +[Apr 27, 2026 - 22:01:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 546 : data loading time 0.00 +[Apr 27, 2026 - 22:01:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 546 : avg forward pass time 13.67 +[Apr 27, 2026 - 22:01:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 546 : backward pass time 12.82 +[Apr 27, 2026 - 22:01:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 546 : optimizer step time 1.45 +[Apr 27, 2026 - 22:01:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 546-------------------- +[Apr 27, 2026 - 22:01:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0188 iter count: 1.0 +[Apr 27, 2026 - 22:01:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0188 iter count: 1.0 +[Apr 27, 2026 - 22:01:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:01:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 22:02:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313208 43.373779 43.202801 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.948242 70.000000 69.839844 +util 95.750000 99.000000 94.000000 +[Apr 27, 2026 - 22:02:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 547 : avg iteration time 48.01 seconds +[Apr 27, 2026 - 22:02:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 547 : data loading time 0.00 +[Apr 27, 2026 - 22:02:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 547 : avg forward pass time 11.28 +[Apr 27, 2026 - 22:02:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 547 : backward pass time 12.80 +[Apr 27, 2026 - 22:02:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 547 : optimizer step time 1.07 +[Apr 27, 2026 - 22:02:18 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 547-------------------- +[Apr 27, 2026 - 22:02:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0178 iter count: 1.0 +[Apr 27, 2026 - 22:02:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0178 iter count: 1.0 +[Apr 27, 2026 - 22:02:18 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:02:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 22:03:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313209 43.373802 43.202801 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.855469 70.058594 69.761719 +util 97.500000 99.000000 93.000000 +[Apr 27, 2026 - 22:03:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 548 : avg iteration time 51.88 seconds +[Apr 27, 2026 - 22:03:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 548 : data loading time 0.00 +[Apr 27, 2026 - 22:03:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 548 : avg forward pass time 13.91 +[Apr 27, 2026 - 22:03:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 548 : backward pass time 11.56 +[Apr 27, 2026 - 22:03:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 548 : optimizer step time 0.98 +[Apr 27, 2026 - 22:03:10 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 548-------------------- +[Apr 27, 2026 - 22:03:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0212 iter count: 1.0 +[Apr 27, 2026 - 22:03:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0212 iter count: 1.0 +[Apr 27, 2026 - 22:03:10 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:03:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 22:04:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313226 43.373802 43.202824 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.997070 70.113281 69.839844 +util 97.750000 100.000000 92.000000 +[Apr 27, 2026 - 22:04:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 549 : avg iteration time 53.38 seconds +[Apr 27, 2026 - 22:04:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 549 : data loading time 0.00 +[Apr 27, 2026 - 22:04:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 549 : avg forward pass time 14.02 +[Apr 27, 2026 - 22:04:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 549 : backward pass time 11.65 +[Apr 27, 2026 - 22:04:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 549 : optimizer step time 1.42 +[Apr 27, 2026 - 22:04:04 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 549-------------------- +[Apr 27, 2026 - 22:04:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0219 iter count: 1.0 +[Apr 27, 2026 - 22:04:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0219 iter count: 1.0 +[Apr 27, 2026 - 22:04:04 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:04:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 22:06:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.312996 43.373825 43.202835 +peak_gpu_mem_gb 129.581032 131.620712 123.567759 +peak_gpu_mem_reserved_gb 133.575195 135.658203 127.451172 +util 61.000000 67.000000 51.000000 +[Apr 27, 2026 - 22:06:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 550 : avg iteration time 168.83 seconds +[Apr 27, 2026 - 22:06:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 550 : data loading time 0.00 +[Apr 27, 2026 - 22:06:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 550 : avg forward pass time 75.05 +[Apr 27, 2026 - 22:06:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 550 : backward pass time 8.50 +[Apr 27, 2026 - 22:06:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 550 : optimizer step time 0.42 +[Apr 27, 2026 - 22:06:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 550-------------------- +[Apr 27, 2026 - 22:06:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 62331.8550 iter count: 1.0 +[Apr 27, 2026 - 22:06:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6047 iter count: 1.0 +[Apr 27, 2026 - 22:06:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 62331.8550iter count: 1.0 +[Apr 27, 2026 - 22:06:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:06:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.4691iter count: 1.0 +[Apr 27, 2026 - 22:06:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.3303 iter count: 1.0 +[Apr 27, 2026 - 22:06:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.0260 iter count: 1.0 +[Apr 27, 2026 - 22:06:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 125521.7656iter count: 1.0 +[Apr 27, 2026 - 22:06:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 14.3039iter count: 1.0 +[Apr 27, 2026 - 22:06:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 1266814.7500iter count: 1.0 +[Apr 27, 2026 - 22:06:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6047iter count: 1.0 +[Apr 27, 2026 - 22:06:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 6.9506 iter count: 1.0 +[Apr 27, 2026 - 22:06:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 1089563.7500iter count: 1.0 +[Apr 27, 2026 - 22:06:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.7159 iter count: 1.0 +[Apr 27, 2026 - 22:06:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 92205.5938iter count: 1.0 +[Apr 27, 2026 - 22:06:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.3309 iter count: 1.0 +[Apr 27, 2026 - 22:06:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 782.0082iter count: 1.0 +[Apr 27, 2026 - 22:06:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.7399 iter count: 1.0 +[Apr 27, 2026 - 22:06:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 96733.3438iter count: 1.0 +[Apr 27, 2026 - 22:06:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0187 iter count: 1.0 +[Apr 27, 2026 - 22:06:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:06:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 22:07:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313168 43.373802 43.202843 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.875000 69.902344 69.859375 +util 99.000000 100.000000 97.000000 +[Apr 27, 2026 - 22:07:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 551 : avg iteration time 48.03 seconds +[Apr 27, 2026 - 22:07:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 551 : data loading time 0.00 +[Apr 27, 2026 - 22:07:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 551 : avg forward pass time 11.94 +[Apr 27, 2026 - 22:07:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 551 : backward pass time 11.58 +[Apr 27, 2026 - 22:07:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 551 : optimizer step time 0.97 +[Apr 27, 2026 - 22:07:40 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 551-------------------- +[Apr 27, 2026 - 22:07:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0185 iter count: 1.0 +[Apr 27, 2026 - 22:07:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0185 iter count: 1.0 +[Apr 27, 2026 - 22:07:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:07:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 22:08:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313169 43.373779 43.202847 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.933594 70.015625 69.820312 +util 99.000000 99.000000 99.000000 +[Apr 27, 2026 - 22:08:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 552 : avg iteration time 50.42 seconds +[Apr 27, 2026 - 22:08:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 552 : data loading time 0.00 +[Apr 27, 2026 - 22:08:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 552 : avg forward pass time 12.59 +[Apr 27, 2026 - 22:08:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 552 : backward pass time 11.54 +[Apr 27, 2026 - 22:08:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 552 : optimizer step time 0.98 +[Apr 27, 2026 - 22:08:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 552-------------------- +[Apr 27, 2026 - 22:08:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0190 iter count: 1.0 +[Apr 27, 2026 - 22:08:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0190 iter count: 1.0 +[Apr 27, 2026 - 22:08:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:08:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 22:09:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313180 43.373779 43.202847 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.928711 69.976562 69.863281 +util 97.750000 100.000000 92.000000 +[Apr 27, 2026 - 22:09:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 553 : avg iteration time 51.93 seconds +[Apr 27, 2026 - 22:09:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 553 : data loading time 0.00 +[Apr 27, 2026 - 22:09:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 553 : avg forward pass time 13.04 +[Apr 27, 2026 - 22:09:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 553 : backward pass time 12.68 +[Apr 27, 2026 - 22:09:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 553 : optimizer step time 1.62 +[Apr 27, 2026 - 22:09:23 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 553-------------------- +[Apr 27, 2026 - 22:09:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0185 iter count: 1.0 +[Apr 27, 2026 - 22:09:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0185 iter count: 1.0 +[Apr 27, 2026 - 22:09:23 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:09:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 22:10:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313414 43.373802 43.202847 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.962891 70.035156 69.882812 +util 96.000000 100.000000 92.000000 +[Apr 27, 2026 - 22:10:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 554 : avg iteration time 51.11 seconds +[Apr 27, 2026 - 22:10:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 554 : data loading time 0.00 +[Apr 27, 2026 - 22:10:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 554 : avg forward pass time 12.87 +[Apr 27, 2026 - 22:10:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 554 : backward pass time 12.17 +[Apr 27, 2026 - 22:10:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 554 : optimizer step time 1.58 +[Apr 27, 2026 - 22:10:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 554-------------------- +[Apr 27, 2026 - 22:10:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0235 iter count: 1.0 +[Apr 27, 2026 - 22:10:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0235 iter count: 1.0 +[Apr 27, 2026 - 22:10:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:10:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 22:13:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313116 43.373795 43.202938 +peak_gpu_mem_gb 129.581032 131.620712 123.567759 +peak_gpu_mem_reserved_gb 133.569824 135.697266 127.550781 +util 68.250000 71.000000 62.000000 +[Apr 27, 2026 - 22:13:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 555 : avg iteration time 172.88 seconds +[Apr 27, 2026 - 22:13:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 555 : data loading time 0.00 +[Apr 27, 2026 - 22:13:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 555 : avg forward pass time 77.89 +[Apr 27, 2026 - 22:13:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 555 : backward pass time 8.38 +[Apr 27, 2026 - 22:13:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 555 : optimizer step time 0.33 +[Apr 27, 2026 - 22:13:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 555-------------------- +[Apr 27, 2026 - 22:13:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 139700.7463 iter count: 1.0 +[Apr 27, 2026 - 22:13:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6235 iter count: 1.0 +[Apr 27, 2026 - 22:13:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 139700.7463iter count: 1.0 +[Apr 27, 2026 - 22:13:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:13:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 3.6956iter count: 1.0 +[Apr 27, 2026 - 22:13:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.5724 iter count: 1.0 +[Apr 27, 2026 - 22:13:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.6702 iter count: 1.0 +[Apr 27, 2026 - 22:13:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 187413.4219iter count: 1.0 +[Apr 27, 2026 - 22:13:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 3.1418 iter count: 1.0 +[Apr 27, 2026 - 22:13:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 4594695.5000iter count: 1.0 +[Apr 27, 2026 - 22:13:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6235iter count: 1.0 +[Apr 27, 2026 - 22:13:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 2.2436 iter count: 1.0 +[Apr 27, 2026 - 22:13:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 88.8673 iter count: 1.0 +[Apr 27, 2026 - 22:13:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.7994 iter count: 1.0 +[Apr 27, 2026 - 22:13:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 1995.9379iter count: 1.0 +[Apr 27, 2026 - 22:13:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.7094 iter count: 1.0 +[Apr 27, 2026 - 22:13:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 12318.1963iter count: 1.0 +[Apr 27, 2026 - 22:13:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.1336 iter count: 1.0 +[Apr 27, 2026 - 22:13:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 3894.0471iter count: 1.0 +[Apr 27, 2026 - 22:13:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0228 iter count: 1.0 +[Apr 27, 2026 - 22:13:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:13:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 22:13:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313189 43.373802 43.202953 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.943359 70.136719 69.839844 +util 93.250000 99.000000 90.000000 +[Apr 27, 2026 - 22:13:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 556 : avg iteration time 52.10 seconds +[Apr 27, 2026 - 22:13:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 556 : data loading time 0.00 +[Apr 27, 2026 - 22:13:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 556 : avg forward pass time 13.40 +[Apr 27, 2026 - 22:13:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 556 : backward pass time 11.56 +[Apr 27, 2026 - 22:13:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 556 : optimizer step time 0.99 +[Apr 27, 2026 - 22:13:59 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 556-------------------- +[Apr 27, 2026 - 22:13:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0254 iter count: 1.0 +[Apr 27, 2026 - 22:13:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0254 iter count: 1.0 +[Apr 27, 2026 - 22:13:59 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:13:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 22:14:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313188 43.373802 43.202938 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.045898 70.113281 69.859375 +util 93.250000 99.000000 90.000000 +[Apr 27, 2026 - 22:14:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 557 : avg iteration time 54.04 seconds +[Apr 27, 2026 - 22:14:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 557 : data loading time 0.00 +[Apr 27, 2026 - 22:14:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 557 : avg forward pass time 14.18 +[Apr 27, 2026 - 22:14:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 557 : backward pass time 12.90 +[Apr 27, 2026 - 22:14:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 557 : optimizer step time 1.25 +[Apr 27, 2026 - 22:14:53 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 557-------------------- +[Apr 27, 2026 - 22:14:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0255 iter count: 1.0 +[Apr 27, 2026 - 22:14:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0255 iter count: 1.0 +[Apr 27, 2026 - 22:14:53 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:14:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 22:15:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313207 43.373878 43.202938 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.948242 69.976562 69.878906 +util 94.250000 99.000000 92.000000 +[Apr 27, 2026 - 22:15:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 558 : avg iteration time 47.37 seconds +[Apr 27, 2026 - 22:15:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 558 : data loading time 0.00 +[Apr 27, 2026 - 22:15:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 558 : avg forward pass time 11.31 +[Apr 27, 2026 - 22:15:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 558 : backward pass time 12.19 +[Apr 27, 2026 - 22:15:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 558 : optimizer step time 0.99 +[Apr 27, 2026 - 22:15:40 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 558-------------------- +[Apr 27, 2026 - 22:15:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0202 iter count: 1.0 +[Apr 27, 2026 - 22:15:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0202 iter count: 1.0 +[Apr 27, 2026 - 22:15:40 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:15:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 22:16:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313215 43.373909 43.202938 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.938477 69.960938 69.878906 +util 98.500000 100.000000 96.000000 +[Apr 27, 2026 - 22:16:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 559 : avg iteration time 49.54 seconds +[Apr 27, 2026 - 22:16:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 559 : data loading time 0.00 +[Apr 27, 2026 - 22:16:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 559 : avg forward pass time 12.65 +[Apr 27, 2026 - 22:16:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 559 : backward pass time 11.56 +[Apr 27, 2026 - 22:16:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 559 : optimizer step time 0.97 +[Apr 27, 2026 - 22:16:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 559-------------------- +[Apr 27, 2026 - 22:16:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0187 iter count: 1.0 +[Apr 27, 2026 - 22:16:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0187 iter count: 1.0 +[Apr 27, 2026 - 22:16:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:16:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 22:19:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313234 43.373882 43.203007 +peak_gpu_mem_gb 129.581032 131.620712 123.567759 +peak_gpu_mem_reserved_gb 133.668457 135.638672 127.765625 +util 74.250000 77.000000 68.000000 +[Apr 27, 2026 - 22:19:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 560 : avg iteration time 171.82 seconds +[Apr 27, 2026 - 22:19:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 560 : data loading time 0.00 +[Apr 27, 2026 - 22:19:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 560 : avg forward pass time 77.42 +[Apr 27, 2026 - 22:19:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 560 : backward pass time 8.34 +[Apr 27, 2026 - 22:19:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 560 : optimizer step time 0.34 +[Apr 27, 2026 - 22:19:22 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 560-------------------- +[Apr 27, 2026 - 22:19:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 114495.0225 iter count: 1.0 +[Apr 27, 2026 - 22:19:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6458 iter count: 1.0 +[Apr 27, 2026 - 22:19:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 114495.0225iter count: 1.0 +[Apr 27, 2026 - 22:19:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:19:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.1576iter count: 1.0 +[Apr 27, 2026 - 22:19:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.8336 iter count: 1.0 +[Apr 27, 2026 - 22:19:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.5455 iter count: 1.0 +[Apr 27, 2026 - 22:19:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 181274.7500iter count: 1.0 +[Apr 27, 2026 - 22:19:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 5.2976 iter count: 1.0 +[Apr 27, 2026 - 22:19:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 3580711.5000iter count: 1.0 +[Apr 27, 2026 - 22:19:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6458iter count: 1.0 +[Apr 27, 2026 - 22:19:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.2806 iter count: 1.0 +[Apr 27, 2026 - 22:19:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 707.0598iter count: 1.0 +[Apr 27, 2026 - 22:19:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 6.5119 iter count: 1.0 +[Apr 27, 2026 - 22:19:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 453039.2812iter count: 1.0 +[Apr 27, 2026 - 22:19:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.3269 iter count: 1.0 +[Apr 27, 2026 - 22:19:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 775.7745iter count: 1.0 +[Apr 27, 2026 - 22:19:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.1801 iter count: 1.0 +[Apr 27, 2026 - 22:19:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 31577.5234iter count: 1.0 +[Apr 27, 2026 - 22:19:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0219 iter count: 1.0 +[Apr 27, 2026 - 22:19:22 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:19:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 22:20:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313298 43.374001 43.203022 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.982910 70.035156 69.898438 +util 98.500000 100.000000 94.000000 +[Apr 27, 2026 - 22:20:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 561 : avg iteration time 52.76 seconds +[Apr 27, 2026 - 22:20:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 561 : data loading time 0.00 +[Apr 27, 2026 - 22:20:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 561 : avg forward pass time 13.59 +[Apr 27, 2026 - 22:20:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 561 : backward pass time 12.73 +[Apr 27, 2026 - 22:20:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 561 : optimizer step time 1.30 +[Apr 27, 2026 - 22:20:14 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 561-------------------- +[Apr 27, 2026 - 22:20:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0249 iter count: 1.0 +[Apr 27, 2026 - 22:20:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0249 iter count: 1.0 +[Apr 27, 2026 - 22:20:14 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:20:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 22:21:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313293 43.374020 43.203007 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.963379 69.996094 69.917969 +util 95.000000 99.000000 93.000000 +[Apr 27, 2026 - 22:21:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 562 : avg iteration time 52.90 seconds +[Apr 27, 2026 - 22:21:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 562 : data loading time 0.00 +[Apr 27, 2026 - 22:21:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 562 : avg forward pass time 14.37 +[Apr 27, 2026 - 22:21:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 562 : backward pass time 11.56 +[Apr 27, 2026 - 22:21:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 562 : optimizer step time 1.00 +[Apr 27, 2026 - 22:21:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 562-------------------- +[Apr 27, 2026 - 22:21:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0258 iter count: 1.0 +[Apr 27, 2026 - 22:21:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0258 iter count: 1.0 +[Apr 27, 2026 - 22:21:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:21:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 22:21:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313302 43.374016 43.203022 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.963379 69.996094 69.904297 +util 93.500000 99.000000 91.000000 +[Apr 27, 2026 - 22:21:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 563 : avg iteration time 47.78 seconds +[Apr 27, 2026 - 22:21:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 563 : data loading time 0.00 +[Apr 27, 2026 - 22:21:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 563 : avg forward pass time 11.37 +[Apr 27, 2026 - 22:21:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 563 : backward pass time 11.57 +[Apr 27, 2026 - 22:21:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 563 : optimizer step time 0.99 +[Apr 27, 2026 - 22:21:55 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 563-------------------- +[Apr 27, 2026 - 22:21:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0214 iter count: 1.0 +[Apr 27, 2026 - 22:21:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0214 iter count: 1.0 +[Apr 27, 2026 - 22:21:55 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:21:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 22:22:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313310 43.374016 43.203060 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.963379 69.996094 69.923828 +util 98.500000 99.000000 97.000000 +[Apr 27, 2026 - 22:22:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 564 : avg iteration time 50.36 seconds +[Apr 27, 2026 - 22:22:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 564 : data loading time 0.00 +[Apr 27, 2026 - 22:22:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 564 : avg forward pass time 12.51 +[Apr 27, 2026 - 22:22:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 564 : backward pass time 11.53 +[Apr 27, 2026 - 22:22:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 564 : optimizer step time 1.03 +[Apr 27, 2026 - 22:22:45 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 564-------------------- +[Apr 27, 2026 - 22:22:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0228 iter count: 1.0 +[Apr 27, 2026 - 22:22:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0228 iter count: 1.0 +[Apr 27, 2026 - 22:22:45 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:22:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 22:25:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.312975 43.374016 43.203045 +peak_gpu_mem_gb 129.581032 131.620712 123.567759 +peak_gpu_mem_reserved_gb 133.600098 135.677734 127.687500 +util 66.750000 73.000000 64.000000 +[Apr 27, 2026 - 22:25:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 565 : avg iteration time 172.26 seconds +[Apr 27, 2026 - 22:25:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 565 : data loading time 0.00 +[Apr 27, 2026 - 22:25:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 565 : avg forward pass time 77.09 +[Apr 27, 2026 - 22:25:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 565 : backward pass time 8.35 +[Apr 27, 2026 - 22:25:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 565 : optimizer step time 0.34 +[Apr 27, 2026 - 22:25:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 565-------------------- +[Apr 27, 2026 - 22:25:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 170023.5869 iter count: 1.0 +[Apr 27, 2026 - 22:25:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6060 iter count: 1.0 +[Apr 27, 2026 - 22:25:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 170023.5869iter count: 1.0 +[Apr 27, 2026 - 22:25:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:25:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.5829iter count: 1.0 +[Apr 27, 2026 - 22:25:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 2.1733 iter count: 1.0 +[Apr 27, 2026 - 22:25:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.3302 iter count: 1.0 +[Apr 27, 2026 - 22:25:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 233331.4062iter count: 1.0 +[Apr 27, 2026 - 22:25:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 77.2175iter count: 1.0 +[Apr 27, 2026 - 22:25:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 2327603.7500iter count: 1.0 +[Apr 27, 2026 - 22:25:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6060iter count: 1.0 +[Apr 27, 2026 - 22:25:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.2741 iter count: 1.0 +[Apr 27, 2026 - 22:25:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 5157.5249iter count: 1.0 +[Apr 27, 2026 - 22:25:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.9883 iter count: 1.0 +[Apr 27, 2026 - 22:25:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 158983.0625iter count: 1.0 +[Apr 27, 2026 - 22:25:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 7.1710 iter count: 1.0 +[Apr 27, 2026 - 22:25:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 1692833.2500iter count: 1.0 +[Apr 27, 2026 - 22:25:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.7104 iter count: 1.0 +[Apr 27, 2026 - 22:25:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 673863.0000iter count: 1.0 +[Apr 27, 2026 - 22:25:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0215 iter count: 1.0 +[Apr 27, 2026 - 22:25:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:25:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 22:26:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.312982 43.374016 43.203049 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.992188 70.074219 69.859375 +util 99.500000 100.000000 99.000000 +[Apr 27, 2026 - 22:26:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 566 : avg iteration time 51.35 seconds +[Apr 27, 2026 - 22:26:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 566 : data loading time 0.00 +[Apr 27, 2026 - 22:26:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 566 : avg forward pass time 13.40 +[Apr 27, 2026 - 22:26:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 566 : backward pass time 11.61 +[Apr 27, 2026 - 22:26:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 566 : optimizer step time 0.98 +[Apr 27, 2026 - 22:26:29 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 566-------------------- +[Apr 27, 2026 - 22:26:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0236 iter count: 1.0 +[Apr 27, 2026 - 22:26:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0236 iter count: 1.0 +[Apr 27, 2026 - 22:26:29 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:26:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 22:27:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313326 43.374016 43.203064 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.972656 70.093750 69.898438 +util 97.000000 99.000000 91.000000 +[Apr 27, 2026 - 22:27:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 567 : avg iteration time 47.32 seconds +[Apr 27, 2026 - 22:27:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 567 : data loading time 0.00 +[Apr 27, 2026 - 22:27:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 567 : avg forward pass time 11.21 +[Apr 27, 2026 - 22:27:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 567 : backward pass time 11.51 +[Apr 27, 2026 - 22:27:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 567 : optimizer step time 1.01 +[Apr 27, 2026 - 22:27:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 567-------------------- +[Apr 27, 2026 - 22:27:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0197 iter count: 1.0 +[Apr 27, 2026 - 22:27:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0197 iter count: 1.0 +[Apr 27, 2026 - 22:27:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:27:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 22:28:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313570 43.374020 43.203041 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.021484 70.093750 69.898438 +util 98.000000 99.000000 95.000000 +[Apr 27, 2026 - 22:28:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 568 : avg iteration time 50.44 seconds +[Apr 27, 2026 - 22:28:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 568 : data loading time 0.00 +[Apr 27, 2026 - 22:28:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 568 : avg forward pass time 12.72 +[Apr 27, 2026 - 22:28:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 568 : backward pass time 12.43 +[Apr 27, 2026 - 22:28:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 568 : optimizer step time 1.01 +[Apr 27, 2026 - 22:28:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 568-------------------- +[Apr 27, 2026 - 22:28:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0182 iter count: 1.0 +[Apr 27, 2026 - 22:28:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0182 iter count: 1.0 +[Apr 27, 2026 - 22:28:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:28:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 22:28:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313584 43.374016 43.203041 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.958008 70.000000 69.878906 +util 94.000000 99.000000 91.000000 +[Apr 27, 2026 - 22:28:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 569 : avg iteration time 49.48 seconds +[Apr 27, 2026 - 22:28:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 569 : data loading time 0.00 +[Apr 27, 2026 - 22:28:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 569 : avg forward pass time 12.71 +[Apr 27, 2026 - 22:28:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 569 : backward pass time 11.55 +[Apr 27, 2026 - 22:28:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 569 : optimizer step time 0.99 +[Apr 27, 2026 - 22:28:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 569-------------------- +[Apr 27, 2026 - 22:28:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0207 iter count: 1.0 +[Apr 27, 2026 - 22:28:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0207 iter count: 1.0 +[Apr 27, 2026 - 22:28:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:28:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 22:31:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313131 43.374054 43.203087 +peak_gpu_mem_gb 129.581032 131.620712 123.567759 +peak_gpu_mem_reserved_gb 133.554199 135.554688 127.566406 +util 67.000000 70.000000 61.000000 +[Apr 27, 2026 - 22:31:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 570 : avg iteration time 174.68 seconds +[Apr 27, 2026 - 22:31:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 570 : data loading time 0.00 +[Apr 27, 2026 - 22:31:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 570 : avg forward pass time 78.86 +[Apr 27, 2026 - 22:31:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 570 : backward pass time 8.34 +[Apr 27, 2026 - 22:31:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 570 : optimizer step time 0.34 +[Apr 27, 2026 - 22:31:51 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 570-------------------- +[Apr 27, 2026 - 22:31:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 370336.6685 iter count: 1.0 +[Apr 27, 2026 - 22:31:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6292 iter count: 1.0 +[Apr 27, 2026 - 22:31:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 370336.6685iter count: 1.0 +[Apr 27, 2026 - 22:31:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:31:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.6402iter count: 1.0 +[Apr 27, 2026 - 22:31:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.4599 iter count: 1.0 +[Apr 27, 2026 - 22:31:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.9329 iter count: 1.0 +[Apr 27, 2026 - 22:31:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 596777.3750iter count: 1.0 +[Apr 27, 2026 - 22:31:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 2.5087 iter count: 1.0 +[Apr 27, 2026 - 22:31:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 7770001.0000iter count: 1.0 +[Apr 27, 2026 - 22:31:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6292iter count: 1.0 +[Apr 27, 2026 - 22:31:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.3720 iter count: 1.0 +[Apr 27, 2026 - 22:31:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 6272.6709iter count: 1.0 +[Apr 27, 2026 - 22:31:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.8199 iter count: 1.0 +[Apr 27, 2026 - 22:31:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 113522.6016iter count: 1.0 +[Apr 27, 2026 - 22:31:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 2.6416 iter count: 1.0 +[Apr 27, 2026 - 22:31:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 197.0089iter count: 1.0 +[Apr 27, 2026 - 22:31:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.5635 iter count: 1.0 +[Apr 27, 2026 - 22:31:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 1245.0728iter count: 1.0 +[Apr 27, 2026 - 22:31:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0201 iter count: 1.0 +[Apr 27, 2026 - 22:31:51 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:31:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 22:32:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313251 43.374054 43.203064 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.055664 70.156250 69.878906 +util 98.000000 100.000000 96.000000 +[Apr 27, 2026 - 22:32:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 571 : avg iteration time 54.98 seconds +[Apr 27, 2026 - 22:32:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 571 : data loading time 0.00 +[Apr 27, 2026 - 22:32:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 571 : avg forward pass time 14.93 +[Apr 27, 2026 - 22:32:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 571 : backward pass time 12.54 +[Apr 27, 2026 - 22:32:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 571 : optimizer step time 1.02 +[Apr 27, 2026 - 22:32:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 571-------------------- +[Apr 27, 2026 - 22:32:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0279 iter count: 1.0 +[Apr 27, 2026 - 22:32:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0279 iter count: 1.0 +[Apr 27, 2026 - 22:32:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:32:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 22:33:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313626 43.374054 43.203045 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.084961 70.093750 70.074219 +util 94.250000 99.000000 92.000000 +[Apr 27, 2026 - 22:33:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 572 : avg iteration time 47.35 seconds +[Apr 27, 2026 - 22:33:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 572 : data loading time 0.00 +[Apr 27, 2026 - 22:33:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 572 : avg forward pass time 11.63 +[Apr 27, 2026 - 22:33:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 572 : backward pass time 11.55 +[Apr 27, 2026 - 22:33:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 572 : optimizer step time 1.00 +[Apr 27, 2026 - 22:33:33 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 572-------------------- +[Apr 27, 2026 - 22:33:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0194 iter count: 1.0 +[Apr 27, 2026 - 22:33:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0194 iter count: 1.0 +[Apr 27, 2026 - 22:33:33 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:33:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 22:34:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313643 43.374100 43.203068 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.967773 70.156250 69.859375 +util 98.000000 99.000000 95.000000 +[Apr 27, 2026 - 22:34:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 573 : avg iteration time 50.93 seconds +[Apr 27, 2026 - 22:34:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 573 : data loading time 0.00 +[Apr 27, 2026 - 22:34:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 573 : avg forward pass time 12.88 +[Apr 27, 2026 - 22:34:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 573 : backward pass time 11.57 +[Apr 27, 2026 - 22:34:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 573 : optimizer step time 0.98 +[Apr 27, 2026 - 22:34:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 573-------------------- +[Apr 27, 2026 - 22:34:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0222 iter count: 1.0 +[Apr 27, 2026 - 22:34:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0222 iter count: 1.0 +[Apr 27, 2026 - 22:34:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:34:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 22:35:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313517 43.374157 43.203045 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.011719 70.156250 69.898438 +util 98.000000 100.000000 93.000000 +[Apr 27, 2026 - 22:35:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 574 : avg iteration time 47.77 seconds +[Apr 27, 2026 - 22:35:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 574 : data loading time 0.00 +[Apr 27, 2026 - 22:35:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 574 : avg forward pass time 11.43 +[Apr 27, 2026 - 22:35:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 574 : backward pass time 11.66 +[Apr 27, 2026 - 22:35:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 574 : optimizer step time 1.43 +[Apr 27, 2026 - 22:35:12 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 574-------------------- +[Apr 27, 2026 - 22:35:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0201 iter count: 1.0 +[Apr 27, 2026 - 22:35:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0201 iter count: 1.0 +[Apr 27, 2026 - 22:35:12 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:35:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 22:38:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313303 43.374157 43.203075 +peak_gpu_mem_gb 129.581032 131.620712 123.567759 +peak_gpu_mem_reserved_gb 133.658691 135.697266 127.863281 +util 68.750000 75.000000 66.000000 +[Apr 27, 2026 - 22:38:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 575 : avg iteration time 176.54 seconds +[Apr 27, 2026 - 22:38:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 575 : data loading time 0.00 +[Apr 27, 2026 - 22:38:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 575 : avg forward pass time 79.72 +[Apr 27, 2026 - 22:38:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 575 : backward pass time 8.40 +[Apr 27, 2026 - 22:38:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 575 : optimizer step time 0.34 +[Apr 27, 2026 - 22:38:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 575-------------------- +[Apr 27, 2026 - 22:38:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 111222.6279 iter count: 1.0 +[Apr 27, 2026 - 22:38:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6563 iter count: 1.0 +[Apr 27, 2026 - 22:38:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 111222.6279iter count: 1.0 +[Apr 27, 2026 - 22:38:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:38:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.7864iter count: 1.0 +[Apr 27, 2026 - 22:38:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.4018 iter count: 1.0 +[Apr 27, 2026 - 22:38:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 7.3540 iter count: 1.0 +[Apr 27, 2026 - 22:38:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 190909.0312iter count: 1.0 +[Apr 27, 2026 - 22:38:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 16.5037iter count: 1.0 +[Apr 27, 2026 - 22:38:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 2441360.5000iter count: 1.0 +[Apr 27, 2026 - 22:38:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6563iter count: 1.0 +[Apr 27, 2026 - 22:38:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.7938 iter count: 1.0 +[Apr 27, 2026 - 22:38:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 1973.6292iter count: 1.0 +[Apr 27, 2026 - 22:38:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.9887 iter count: 1.0 +[Apr 27, 2026 - 22:38:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 159118.8281iter count: 1.0 +[Apr 27, 2026 - 22:38:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 5.7896 iter count: 1.0 +[Apr 27, 2026 - 22:38:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 106849.0234iter count: 1.0 +[Apr 27, 2026 - 22:38:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 3.6862 iter count: 1.0 +[Apr 27, 2026 - 22:38:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 1591.4613iter count: 1.0 +[Apr 27, 2026 - 22:38:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0254 iter count: 1.0 +[Apr 27, 2026 - 22:38:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:38:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 22:39:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313424 43.374172 43.203068 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.850586 69.898438 69.746094 +util 98.000000 100.000000 94.000000 +[Apr 27, 2026 - 22:39:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 576 : avg iteration time 54.47 seconds +[Apr 27, 2026 - 22:39:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 576 : data loading time 0.00 +[Apr 27, 2026 - 22:39:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 576 : avg forward pass time 14.54 +[Apr 27, 2026 - 22:39:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 576 : backward pass time 11.54 +[Apr 27, 2026 - 22:39:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 576 : optimizer step time 0.98 +[Apr 27, 2026 - 22:39:03 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 576-------------------- +[Apr 27, 2026 - 22:39:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0193 iter count: 1.0 +[Apr 27, 2026 - 22:39:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0193 iter count: 1.0 +[Apr 27, 2026 - 22:39:03 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:39:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 22:39:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313483 43.374157 43.203064 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.958008 70.132812 69.785156 +util 98.250000 100.000000 97.000000 +[Apr 27, 2026 - 22:39:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 577 : avg iteration time 48.74 seconds +[Apr 27, 2026 - 22:39:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 577 : data loading time 0.00 +[Apr 27, 2026 - 22:39:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 577 : avg forward pass time 11.58 +[Apr 27, 2026 - 22:39:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 577 : backward pass time 12.47 +[Apr 27, 2026 - 22:39:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 577 : optimizer step time 1.57 +[Apr 27, 2026 - 22:39:52 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 577-------------------- +[Apr 27, 2026 - 22:39:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0174 iter count: 1.0 +[Apr 27, 2026 - 22:39:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0174 iter count: 1.0 +[Apr 27, 2026 - 22:39:52 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:39:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 22:40:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313489 43.374180 43.203064 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.909180 70.113281 69.609375 +util 98.000000 99.000000 97.000000 +[Apr 27, 2026 - 22:40:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 578 : avg iteration time 50.21 seconds +[Apr 27, 2026 - 22:40:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 578 : data loading time 0.00 +[Apr 27, 2026 - 22:40:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 578 : avg forward pass time 12.74 +[Apr 27, 2026 - 22:40:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 578 : backward pass time 12.19 +[Apr 27, 2026 - 22:40:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 578 : optimizer step time 0.98 +[Apr 27, 2026 - 22:40:42 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 578-------------------- +[Apr 27, 2026 - 22:40:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0182 iter count: 1.0 +[Apr 27, 2026 - 22:40:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0182 iter count: 1.0 +[Apr 27, 2026 - 22:40:42 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:40:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 22:41:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313555 43.374180 43.203064 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.860352 70.035156 69.492188 +util 98.500000 100.000000 96.000000 +[Apr 27, 2026 - 22:41:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 579 : avg iteration time 49.28 seconds +[Apr 27, 2026 - 22:41:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 579 : data loading time 0.00 +[Apr 27, 2026 - 22:41:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 579 : avg forward pass time 12.57 +[Apr 27, 2026 - 22:41:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 579 : backward pass time 11.59 +[Apr 27, 2026 - 22:41:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 579 : optimizer step time 0.98 +[Apr 27, 2026 - 22:41:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 579-------------------- +[Apr 27, 2026 - 22:41:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0210 iter count: 1.0 +[Apr 27, 2026 - 22:41:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0210 iter count: 1.0 +[Apr 27, 2026 - 22:41:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:41:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 22:44:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313539 43.374409 43.203312 +peak_gpu_mem_gb 129.581032 131.620712 123.567759 +peak_gpu_mem_reserved_gb 133.882324 135.671875 128.625000 +util 69.500000 72.000000 63.000000 +[Apr 27, 2026 - 22:44:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 580 : avg iteration time 175.97 seconds +[Apr 27, 2026 - 22:44:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 580 : data loading time 0.00 +[Apr 27, 2026 - 22:44:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 580 : avg forward pass time 79.44 +[Apr 27, 2026 - 22:44:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 580 : backward pass time 8.38 +[Apr 27, 2026 - 22:44:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 580 : optimizer step time 0.34 +[Apr 27, 2026 - 22:44:27 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 580-------------------- +[Apr 27, 2026 - 22:44:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 180252.1187 iter count: 1.0 +[Apr 27, 2026 - 22:44:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5250 iter count: 1.0 +[Apr 27, 2026 - 22:44:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 180252.1187iter count: 1.0 +[Apr 27, 2026 - 22:44:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:44:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.6213iter count: 1.0 +[Apr 27, 2026 - 22:44:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.2484 iter count: 1.0 +[Apr 27, 2026 - 22:44:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.0001 iter count: 1.0 +[Apr 27, 2026 - 22:44:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 416394.6250iter count: 1.0 +[Apr 27, 2026 - 22:44:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 12.1429iter count: 1.0 +[Apr 27, 2026 - 22:44:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 8888535.0000iter count: 1.0 +[Apr 27, 2026 - 22:44:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5250iter count: 1.0 +[Apr 27, 2026 - 22:44:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.7640 iter count: 1.0 +[Apr 27, 2026 - 22:44:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 1859.4182iter count: 1.0 +[Apr 27, 2026 - 22:44:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 7.3675 iter count: 1.0 +[Apr 27, 2026 - 22:44:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 2508149.7500iter count: 1.0 +[Apr 27, 2026 - 22:44:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.9144 iter count: 1.0 +[Apr 27, 2026 - 22:44:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 18559.2207iter count: 1.0 +[Apr 27, 2026 - 22:44:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.4165 iter count: 1.0 +[Apr 27, 2026 - 22:44:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 50667.5156iter count: 1.0 +[Apr 27, 2026 - 22:44:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0184 iter count: 1.0 +[Apr 27, 2026 - 22:44:27 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:44:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 22:45:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313649 43.374382 43.203312 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.880371 69.884766 69.878906 +util 98.000000 100.000000 93.000000 +[Apr 27, 2026 - 22:45:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 581 : avg iteration time 51.99 seconds +[Apr 27, 2026 - 22:45:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 581 : data loading time 0.00 +[Apr 27, 2026 - 22:45:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 581 : avg forward pass time 13.30 +[Apr 27, 2026 - 22:45:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 581 : backward pass time 11.78 +[Apr 27, 2026 - 22:45:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 581 : optimizer step time 1.50 +[Apr 27, 2026 - 22:45:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 581-------------------- +[Apr 27, 2026 - 22:45:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0197 iter count: 1.0 +[Apr 27, 2026 - 22:45:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0197 iter count: 1.0 +[Apr 27, 2026 - 22:45:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:45:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 22:46:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313754 43.374386 43.203316 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.875488 69.904297 69.859375 +util 99.000000 99.000000 99.000000 +[Apr 27, 2026 - 22:46:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 582 : avg iteration time 47.48 seconds +[Apr 27, 2026 - 22:46:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 582 : data loading time 0.00 +[Apr 27, 2026 - 22:46:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 582 : avg forward pass time 11.12 +[Apr 27, 2026 - 22:46:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 582 : backward pass time 12.64 +[Apr 27, 2026 - 22:46:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 582 : optimizer step time 1.03 +[Apr 27, 2026 - 22:46:07 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 582-------------------- +[Apr 27, 2026 - 22:46:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0179 iter count: 1.0 +[Apr 27, 2026 - 22:46:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0179 iter count: 1.0 +[Apr 27, 2026 - 22:46:07 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:46:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 22:46:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313757 43.374386 43.203316 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.982910 70.138672 69.839844 +util 98.250000 99.000000 96.000000 +[Apr 27, 2026 - 22:46:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 583 : avg iteration time 49.75 seconds +[Apr 27, 2026 - 22:46:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 583 : data loading time 0.00 +[Apr 27, 2026 - 22:46:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 583 : avg forward pass time 12.82 +[Apr 27, 2026 - 22:46:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 583 : backward pass time 11.54 +[Apr 27, 2026 - 22:46:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 583 : optimizer step time 1.01 +[Apr 27, 2026 - 22:46:56 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 583-------------------- +[Apr 27, 2026 - 22:46:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0211 iter count: 1.0 +[Apr 27, 2026 - 22:46:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0211 iter count: 1.0 +[Apr 27, 2026 - 22:46:56 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:46:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 22:47:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313930 43.374386 43.203293 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.899902 69.982422 69.859375 +util 95.000000 99.000000 91.000000 +[Apr 27, 2026 - 22:47:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 584 : avg iteration time 50.05 seconds +[Apr 27, 2026 - 22:47:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 584 : data loading time 0.00 +[Apr 27, 2026 - 22:47:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 584 : avg forward pass time 12.43 +[Apr 27, 2026 - 22:47:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 584 : backward pass time 11.55 +[Apr 27, 2026 - 22:47:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 584 : optimizer step time 0.98 +[Apr 27, 2026 - 22:47:46 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 584-------------------- +[Apr 27, 2026 - 22:47:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0158 iter count: 1.0 +[Apr 27, 2026 - 22:47:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0158 iter count: 1.0 +[Apr 27, 2026 - 22:47:46 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:47:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 22:50:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313729 43.374390 43.203293 +peak_gpu_mem_gb 129.581032 131.620712 123.567759 +peak_gpu_mem_reserved_gb 133.614746 135.677734 127.667969 +util 65.500000 72.000000 63.000000 +[Apr 27, 2026 - 22:50:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 585 : avg iteration time 176.23 seconds +[Apr 27, 2026 - 22:50:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 585 : data loading time 0.00 +[Apr 27, 2026 - 22:50:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 585 : avg forward pass time 79.59 +[Apr 27, 2026 - 22:50:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 585 : backward pass time 8.37 +[Apr 27, 2026 - 22:50:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 585 : optimizer step time 0.34 +[Apr 27, 2026 - 22:50:43 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 585-------------------- +[Apr 27, 2026 - 22:50:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 43037.4014 iter count: 1.0 +[Apr 27, 2026 - 22:50:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6954 iter count: 1.0 +[Apr 27, 2026 - 22:50:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 43037.4014iter count: 1.0 +[Apr 27, 2026 - 22:50:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:50:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.4339iter count: 1.0 +[Apr 27, 2026 - 22:50:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.0556 iter count: 1.0 +[Apr 27, 2026 - 22:50:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 6.7717 iter count: 1.0 +[Apr 27, 2026 - 22:50:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 82998.9609iter count: 1.0 +[Apr 27, 2026 - 22:50:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 8.2581 iter count: 1.0 +[Apr 27, 2026 - 22:50:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 761841.1250iter count: 1.0 +[Apr 27, 2026 - 22:50:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6954iter count: 1.0 +[Apr 27, 2026 - 22:50:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.9182 iter count: 1.0 +[Apr 27, 2026 - 22:50:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 2531.0259iter count: 1.0 +[Apr 27, 2026 - 22:50:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 5.5201 iter count: 1.0 +[Apr 27, 2026 - 22:50:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 62324.2500iter count: 1.0 +[Apr 27, 2026 - 22:50:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 6.1561 iter count: 1.0 +[Apr 27, 2026 - 22:50:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 222397.9062iter count: 1.0 +[Apr 27, 2026 - 22:50:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 6.0316 iter count: 1.0 +[Apr 27, 2026 - 22:50:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 173359.5000iter count: 1.0 +[Apr 27, 2026 - 22:50:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0167 iter count: 1.0 +[Apr 27, 2026 - 22:50:43 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:50:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 22:51:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313732 43.374390 43.203304 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.963379 70.041016 69.898438 +util 99.000000 99.000000 99.000000 +[Apr 27, 2026 - 22:51:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 586 : avg iteration time 48.57 seconds +[Apr 27, 2026 - 22:51:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 586 : data loading time 0.00 +[Apr 27, 2026 - 22:51:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 586 : avg forward pass time 11.82 +[Apr 27, 2026 - 22:51:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 586 : backward pass time 11.52 +[Apr 27, 2026 - 22:51:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 586 : optimizer step time 1.01 +[Apr 27, 2026 - 22:51:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 586-------------------- +[Apr 27, 2026 - 22:51:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0143 iter count: 1.0 +[Apr 27, 2026 - 22:51:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0143 iter count: 1.0 +[Apr 27, 2026 - 22:51:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:51:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 22:52:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313804 43.374458 43.203297 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.007324 70.138672 69.820312 +util 98.500000 99.000000 97.000000 +[Apr 27, 2026 - 22:52:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 587 : avg iteration time 47.58 seconds +[Apr 27, 2026 - 22:52:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 587 : data loading time 0.00 +[Apr 27, 2026 - 22:52:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 587 : avg forward pass time 11.14 +[Apr 27, 2026 - 22:52:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 587 : backward pass time 12.67 +[Apr 27, 2026 - 22:52:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 587 : optimizer step time 1.07 +[Apr 27, 2026 - 22:52:19 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 587-------------------- +[Apr 27, 2026 - 22:52:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0139 iter count: 1.0 +[Apr 27, 2026 - 22:52:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0139 iter count: 1.0 +[Apr 27, 2026 - 22:52:19 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:52:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 22:53:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313811 43.374451 43.203297 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.870605 69.982422 69.820312 +util 99.000000 99.000000 99.000000 +[Apr 27, 2026 - 22:53:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 588 : avg iteration time 49.02 seconds +[Apr 27, 2026 - 22:53:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 588 : data loading time 0.00 +[Apr 27, 2026 - 22:53:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 588 : avg forward pass time 12.46 +[Apr 27, 2026 - 22:53:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 588 : backward pass time 11.58 +[Apr 27, 2026 - 22:53:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 588 : optimizer step time 0.98 +[Apr 27, 2026 - 22:53:08 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 588-------------------- +[Apr 27, 2026 - 22:53:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0142 iter count: 1.0 +[Apr 27, 2026 - 22:53:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0142 iter count: 1.0 +[Apr 27, 2026 - 22:53:08 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:53:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 22:53:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313818 43.374485 43.203300 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 71.345215 72.730469 69.820312 +util 98.750000 100.000000 97.000000 +[Apr 27, 2026 - 22:53:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 589 : avg iteration time 49.99 seconds +[Apr 27, 2026 - 22:53:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 589 : data loading time 0.00 +[Apr 27, 2026 - 22:53:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 589 : avg forward pass time 12.56 +[Apr 27, 2026 - 22:53:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 589 : backward pass time 11.54 +[Apr 27, 2026 - 22:53:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 589 : optimizer step time 1.01 +[Apr 27, 2026 - 22:53:58 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 589-------------------- +[Apr 27, 2026 - 22:53:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0127 iter count: 1.0 +[Apr 27, 2026 - 22:53:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0127 iter count: 1.0 +[Apr 27, 2026 - 22:53:58 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:53:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 22:56:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313831 43.374493 43.203331 +peak_gpu_mem_gb 129.581032 131.620712 123.567759 +peak_gpu_mem_reserved_gb 133.613770 135.677734 127.667969 +util 70.750000 77.000000 68.000000 +[Apr 27, 2026 - 22:56:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 590 : avg iteration time 169.30 seconds +[Apr 27, 2026 - 22:56:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 590 : data loading time 0.00 +[Apr 27, 2026 - 22:56:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 590 : avg forward pass time 76.13 +[Apr 27, 2026 - 22:56:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 590 : backward pass time 8.39 +[Apr 27, 2026 - 22:56:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 590 : optimizer step time 0.34 +[Apr 27, 2026 - 22:56:47 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 590-------------------- +[Apr 27, 2026 - 22:56:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 31279.7939 iter count: 1.0 +[Apr 27, 2026 - 22:56:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.5999 iter count: 1.0 +[Apr 27, 2026 - 22:56:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 31279.7939iter count: 1.0 +[Apr 27, 2026 - 22:56:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:56:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.4192iter count: 1.0 +[Apr 27, 2026 - 22:56:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 1.9553 iter count: 1.0 +[Apr 27, 2026 - 22:56:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 6.5523 iter count: 1.0 +[Apr 27, 2026 - 22:56:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 44493.6289iter count: 1.0 +[Apr 27, 2026 - 22:56:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 49.9303iter count: 1.0 +[Apr 27, 2026 - 22:56:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 491229.9375iter count: 1.0 +[Apr 27, 2026 - 22:56:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.5999iter count: 1.0 +[Apr 27, 2026 - 22:56:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 2.6954 iter count: 1.0 +[Apr 27, 2026 - 22:56:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 219.3852iter count: 1.0 +[Apr 27, 2026 - 22:56:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 4.6418 iter count: 1.0 +[Apr 27, 2026 - 22:56:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 10759.7891iter count: 1.0 +[Apr 27, 2026 - 22:56:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.9941 iter count: 1.0 +[Apr 27, 2026 - 22:56:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 21770.1816iter count: 1.0 +[Apr 27, 2026 - 22:56:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 4.5741 iter count: 1.0 +[Apr 27, 2026 - 22:56:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 9396.7842iter count: 1.0 +[Apr 27, 2026 - 22:56:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0134 iter count: 1.0 +[Apr 27, 2026 - 22:56:47 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:56:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 22:57:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313833 43.374477 43.203331 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 71.398438 72.808594 69.898438 +util 96.750000 99.000000 94.000000 +[Apr 27, 2026 - 22:57:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 591 : avg iteration time 51.37 seconds +[Apr 27, 2026 - 22:57:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 591 : data loading time 0.00 +[Apr 27, 2026 - 22:57:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 591 : avg forward pass time 13.27 +[Apr 27, 2026 - 22:57:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 591 : backward pass time 11.54 +[Apr 27, 2026 - 22:57:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 591 : optimizer step time 1.19 +[Apr 27, 2026 - 22:57:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 591-------------------- +[Apr 27, 2026 - 22:57:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0158 iter count: 1.0 +[Apr 27, 2026 - 22:57:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0158 iter count: 1.0 +[Apr 27, 2026 - 22:57:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:57:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 22:58:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313828 43.374496 43.203335 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 71.408203 72.828125 69.937500 +util 98.250000 100.000000 95.000000 +[Apr 27, 2026 - 22:58:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 592 : avg iteration time 52.51 seconds +[Apr 27, 2026 - 22:58:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 592 : data loading time 0.00 +[Apr 27, 2026 - 22:58:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 592 : avg forward pass time 14.17 +[Apr 27, 2026 - 22:58:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 592 : backward pass time 11.59 +[Apr 27, 2026 - 22:58:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 592 : optimizer step time 0.99 +[Apr 27, 2026 - 22:58:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 592-------------------- +[Apr 27, 2026 - 22:58:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0180 iter count: 1.0 +[Apr 27, 2026 - 22:58:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0180 iter count: 1.0 +[Apr 27, 2026 - 22:58:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:58:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 22:59:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313842 43.374527 43.203335 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 71.427734 72.925781 69.800781 +util 99.250000 100.000000 99.000000 +[Apr 27, 2026 - 22:59:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 593 : avg iteration time 50.23 seconds +[Apr 27, 2026 - 22:59:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 593 : data loading time 0.01 +[Apr 27, 2026 - 22:59:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 593 : avg forward pass time 12.57 +[Apr 27, 2026 - 22:59:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 593 : backward pass time 11.54 +[Apr 27, 2026 - 22:59:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 593 : optimizer step time 0.99 +[Apr 27, 2026 - 22:59:21 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 593-------------------- +[Apr 27, 2026 - 22:59:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0136 iter count: 1.0 +[Apr 27, 2026 - 22:59:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0136 iter count: 1.0 +[Apr 27, 2026 - 22:59:21 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 22:59:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 23:00:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313833 43.374504 43.203335 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 71.393555 72.789062 69.957031 +util 98.500000 100.000000 97.000000 +[Apr 27, 2026 - 23:00:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 594 : avg iteration time 47.49 seconds +[Apr 27, 2026 - 23:00:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 594 : data loading time 0.00 +[Apr 27, 2026 - 23:00:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 594 : avg forward pass time 11.32 +[Apr 27, 2026 - 23:00:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 594 : backward pass time 11.63 +[Apr 27, 2026 - 23:00:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 594 : optimizer step time 1.40 +[Apr 27, 2026 - 23:00:09 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 594-------------------- +[Apr 27, 2026 - 23:00:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0153 iter count: 1.0 +[Apr 27, 2026 - 23:00:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0153 iter count: 1.0 +[Apr 27, 2026 - 23:00:09 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 23:00:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 23:03:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313832 43.374527 43.203342 +peak_gpu_mem_gb 129.581032 131.620712 123.567759 +peak_gpu_mem_reserved_gb 133.590332 135.658203 127.667969 +util 70.000000 73.000000 64.000000 +[Apr 27, 2026 - 23:03:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 595 : avg iteration time 172.65 seconds +[Apr 27, 2026 - 23:03:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 595 : data loading time 0.00 +[Apr 27, 2026 - 23:03:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 595 : avg forward pass time 77.79 +[Apr 27, 2026 - 23:03:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 595 : backward pass time 8.38 +[Apr 27, 2026 - 23:03:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 595 : optimizer step time 0.34 +[Apr 27, 2026 - 23:03:01 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 595-------------------- +[Apr 27, 2026 - 23:03:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 616925.5264 iter count: 1.0 +[Apr 27, 2026 - 23:03:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6028 iter count: 1.0 +[Apr 27, 2026 - 23:03:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 616925.5264iter count: 1.0 +[Apr 27, 2026 - 23:03:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 23:03:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 5.1445iter count: 1.0 +[Apr 27, 2026 - 23:03:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 2.2848 iter count: 1.0 +[Apr 27, 2026 - 23:03:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 8.5444 iter count: 1.0 +[Apr 27, 2026 - 23:03:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 1376514.8750iter count: 1.0 +[Apr 27, 2026 - 23:03:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 96.4971iter count: 1.0 +[Apr 27, 2026 - 23:03:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 26395980.0000iter count: 1.0 +[Apr 27, 2026 - 23:03:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6028iter count: 1.0 +[Apr 27, 2026 - 23:03:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 3.0954 iter count: 1.0 +[Apr 27, 2026 - 23:03:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 488.1889iter count: 1.0 +[Apr 27, 2026 - 23:03:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 7.2357 iter count: 1.0 +[Apr 27, 2026 - 23:03:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 1926911.2500iter count: 1.0 +[Apr 27, 2026 - 23:03:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 4.9163 iter count: 1.0 +[Apr 27, 2026 - 23:03:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 18630.6875iter count: 1.0 +[Apr 27, 2026 - 23:03:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 5.7180 iter count: 1.0 +[Apr 27, 2026 - 23:03:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 92594.0234iter count: 1.0 +[Apr 27, 2026 - 23:03:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0146 iter count: 1.0 +[Apr 27, 2026 - 23:03:01 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 23:03:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 23:03:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313837 43.374546 43.203342 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.026367 70.078125 69.917969 +util 98.000000 100.000000 94.000000 +[Apr 27, 2026 - 23:03:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 596 : avg iteration time 49.01 seconds +[Apr 27, 2026 - 23:03:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 596 : data loading time 0.00 +[Apr 27, 2026 - 23:03:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 596 : avg forward pass time 11.98 +[Apr 27, 2026 - 23:03:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 596 : backward pass time 11.55 +[Apr 27, 2026 - 23:03:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 596 : optimizer step time 1.02 +[Apr 27, 2026 - 23:03:50 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 596-------------------- +[Apr 27, 2026 - 23:03:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0129 iter count: 1.0 +[Apr 27, 2026 - 23:03:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0129 iter count: 1.0 +[Apr 27, 2026 - 23:03:50 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 23:03:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 23:04:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313848 43.374546 43.203346 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.943359 70.000000 69.898438 +util 99.000000 99.000000 99.000000 +[Apr 27, 2026 - 23:04:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 597 : avg iteration time 47.66 seconds +[Apr 27, 2026 - 23:04:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 597 : data loading time 0.00 +[Apr 27, 2026 - 23:04:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 597 : avg forward pass time 11.20 +[Apr 27, 2026 - 23:04:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 597 : backward pass time 12.63 +[Apr 27, 2026 - 23:04:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 597 : optimizer step time 1.09 +[Apr 27, 2026 - 23:04:38 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 597-------------------- +[Apr 27, 2026 - 23:04:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0126 iter count: 1.0 +[Apr 27, 2026 - 23:04:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0126 iter count: 1.0 +[Apr 27, 2026 - 23:04:38 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 23:04:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 23:05:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313905 43.374546 43.203342 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.987305 70.078125 69.937500 +util 93.500000 99.000000 91.000000 +[Apr 27, 2026 - 23:05:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 598 : avg iteration time 52.42 seconds +[Apr 27, 2026 - 23:05:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 598 : data loading time 0.00 +[Apr 27, 2026 - 23:05:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 598 : avg forward pass time 14.17 +[Apr 27, 2026 - 23:05:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 598 : backward pass time 11.58 +[Apr 27, 2026 - 23:05:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 598 : optimizer step time 0.98 +[Apr 27, 2026 - 23:05:30 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 598-------------------- +[Apr 27, 2026 - 23:05:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0139 iter count: 1.0 +[Apr 27, 2026 - 23:05:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0139 iter count: 1.0 +[Apr 27, 2026 - 23:05:30 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 23:05:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 23:06:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313910 43.374546 43.203365 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 69.987305 70.078125 69.937500 +util 97.500000 100.000000 95.000000 +[Apr 27, 2026 - 23:06:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 599 : avg iteration time 53.04 seconds +[Apr 27, 2026 - 23:06:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 599 : data loading time 0.00 +[Apr 27, 2026 - 23:06:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 599 : avg forward pass time 13.87 +[Apr 27, 2026 - 23:06:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 599 : backward pass time 11.59 +[Apr 27, 2026 - 23:06:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 599 : optimizer step time 1.35 +[Apr 27, 2026 - 23:06:24 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 599-------------------- +[Apr 27, 2026 - 23:06:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0128 iter count: 1.0 +[Apr 27, 2026 - 23:06:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0128 iter count: 1.0 +[Apr 27, 2026 - 23:06:24 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 23:06:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.01 seconds +[Apr 27, 2026 - 23:09:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.313542 43.374622 43.203354 +peak_gpu_mem_gb 129.581032 131.620712 123.567759 +peak_gpu_mem_reserved_gb 133.575195 135.638672 127.587891 +util 72.250000 75.000000 66.000000 +[Apr 27, 2026 - 23:09:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 600 : avg iteration time 172.90 seconds +[Apr 27, 2026 - 23:09:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 600 : data loading time 0.00 +[Apr 27, 2026 - 23:09:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 600 : avg forward pass time 77.92 +[Apr 27, 2026 - 23:09:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 600 : backward pass time 8.39 +[Apr 27, 2026 - 23:09:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 600 : optimizer step time 0.34 +[Apr 27, 2026 - 23:09:16 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 600-------------------- +[Apr 27, 2026 - 23:09:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 27807.3857 iter count: 1.0 +[Apr 27, 2026 - 23:09:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss: 0.6857 iter count: 1.0 +[Apr 27, 2026 - 23:09:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_weighted: 27807.3857iter count: 1.0 +[Apr 27, 2026 - 23:09:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 23:09:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_mean: 4.2988iter count: 1.0 +[Apr 27, 2026 - 23:09:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_min: 0.3983 iter count: 1.0 +[Apr 27, 2026 - 23:09:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_max: 6.1701 iter count: 1.0 +[Apr 27, 2026 - 23:09:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_mean: 40231.3984iter count: 1.0 +[Apr 27, 2026 - 23:09:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_min: 2.2181 iter count: 1.0 +[Apr 27, 2026 - 23:09:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_max: 228703.4375iter count: 1.0 +[Apr 27, 2026 - 23:09:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_vsd_loss_unweighted: 0.6857iter count: 1.0 +[Apr 27, 2026 - 23:09:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r0: 4.5171 iter count: 1.0 +[Apr 27, 2026 - 23:09:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r0: 8385.1045iter count: 1.0 +[Apr 27, 2026 - 23:09:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r1: 3.3441 iter count: 1.0 +[Apr 27, 2026 - 23:09:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r1: 802.7992iter count: 1.0 +[Apr 27, 2026 - 23:09:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r2: 3.8476 iter count: 1.0 +[Apr 27, 2026 - 23:09:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r2: 2197.6726iter count: 1.0 +[Apr 27, 2026 - 23:09:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_sync_c_r3: 2.9943 iter count: 1.0 +[Apr 27, 2026 - 23:09:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_reward_weight_r3: 398.8344iter count: 1.0 +[Apr 27, 2026 - 23:09:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0165 iter count: 1.0 +[Apr 27, 2026 - 23:09:16 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[Apr 27, 2026 - 23:09:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 14.92 seconds +[Apr 27, 2026 - 23:09:31 | INFO | fastgen.trainer:save_checkpoint:266 ] Saving checkpoint iteration 600 +[Apr 27, 2026 - 23:09:31 | INFO | fastgen.utils.checkpointer:save:301 ] Saving FSDP model to prefix /tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints/0000600 +[Apr 27, 2026 - 23:09:46 | SUCCESS | fastgen.utils.checkpointer:save:329 ] Model saved at iteration 600 +[Apr 27, 2026 - 23:09:46 | INFO | fastgen.trainer:validate:358 ] Validation iteration 600 +[Apr 27, 2026 - 23:09:46 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 23:09:49 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 2.2s +[Apr 27, 2026 - 23:09:52 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 0: sync_c=5.887 +[Apr 27, 2026 - 23:09:52 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 23:09:54 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.8s +[Apr 27, 2026 - 23:09:57 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 1: sync_c=5.520 +[Apr 27, 2026 - 23:09:57 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 23:09:59 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.8s +[Apr 27, 2026 - 23:10:02 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 2: sync_c=4.357 +[Apr 27, 2026 - 23:10:02 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 23:10:04 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.8s +[Apr 27, 2026 - 23:10:07 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 3: sync_c=3.957 +[Apr 27, 2026 - 23:10:08 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 23:10:10 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.9s +[Apr 27, 2026 - 23:10:13 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 4: sync_c=4.839 +[Apr 27, 2026 - 23:10:13 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 23:10:15 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.8s +[Apr 27, 2026 - 23:10:18 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 5: sync_c=6.523 +[Apr 27, 2026 - 23:10:18 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 23:10:20 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.8s +[Apr 27, 2026 - 23:10:23 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 6: sync_c=6.311 +[Apr 27, 2026 - 23:10:23 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 23:10:25 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.8s +[Apr 27, 2026 - 23:10:28 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 7: sync_c=4.898 +[Apr 27, 2026 - 23:10:28 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 23:10:30 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.8s +[Apr 27, 2026 - 23:10:33 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 8: sync_c=5.596 +[Apr 27, 2026 - 23:10:33 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] Starting CausVid AR inference (B=1, T=21, steps=2) +[Apr 27, 2026 - 23:10:35 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:221 ] [val] AR inference done in 1.8s +[Apr 27, 2026 - 23:10:38 | INFO | fastgen.callbacks.wandb:on_validation_step_end:735 ] [SyncEval] val sample 9: sync_c=3.236 +[Apr 27, 2026 - 23:10:39 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging val0 stats at iteration 600-------------------- +[Apr 27, 2026 - 23:10:39 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0000 iter count: 10.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[Apr 27, 2026 - 23:10:55 | INFO | fastgen.callbacks.wandb:on_validation_end:763 ] Logged 10 val videos at iteration 600 +[Apr 27, 2026 - 23:10:55 | INFO | fastgen.callbacks.wandb:on_validation_end:769 ] [SyncEval] val0 mean sync_c: 5.112 (n=10) +[Apr 27, 2026 - 23:11:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.314685 43.375111 43.202850 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.065430 70.292969 69.937500 +util 93.750000 100.000000 91.000000 +[Apr 27, 2026 - 23:11:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 601 : avg iteration time 147.66 seconds +[Apr 27, 2026 - 23:11:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 601 : data loading time 0.00 +[Apr 27, 2026 - 23:11:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 601 : avg forward pass time 12.23 +[Apr 27, 2026 - 23:11:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 601 : backward pass time 11.59 +[Apr 27, 2026 - 23:11:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 601 : optimizer step time 0.97 +[Apr 27, 2026 - 23:11:44 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 601-------------------- +[Apr 27, 2026 - 23:11:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0144 iter count: 1.0 +[Apr 27, 2026 - 23:11:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0144 iter count: 1.0 +[Apr 27, 2026 - 23:11:44 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 23:11:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 23:12:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.314685 43.375111 43.202850 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.006836 70.292969 69.839844 +util 95.250000 100.000000 93.000000 +[Apr 27, 2026 - 23:12:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 602 : avg iteration time 47.42 seconds +[Apr 27, 2026 - 23:12:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 602 : data loading time 0.00 +[Apr 27, 2026 - 23:12:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 602 : avg forward pass time 11.13 +[Apr 27, 2026 - 23:12:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 602 : backward pass time 11.59 +[Apr 27, 2026 - 23:12:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 602 : optimizer step time 0.98 +[Apr 27, 2026 - 23:12:31 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 602-------------------- +[Apr 27, 2026 - 23:12:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0140 iter count: 1.0 +[Apr 27, 2026 - 23:12:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0140 iter count: 1.0 +[Apr 27, 2026 - 23:12:31 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 23:12:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds +[Apr 27, 2026 - 23:13:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.314808 43.375111 43.202854 +peak_gpu_mem_gb 67.820088 67.859751 67.806867 +peak_gpu_mem_reserved_gb 70.075195 70.371094 69.917969 +util 95.750000 100.000000 94.000000 +[Apr 27, 2026 - 23:13:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 603 : avg iteration time 48.41 seconds +[Apr 27, 2026 - 23:13:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 603 : data loading time 0.00 +[Apr 27, 2026 - 23:13:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 603 : avg forward pass time 11.54 +[Apr 27, 2026 - 23:13:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 603 : backward pass time 12.50 +[Apr 27, 2026 - 23:13:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 603 : optimizer step time 1.23 +[Apr 27, 2026 - 23:13:20 | INFO | fastgen.callbacks.wandb:log_stats:630 ] logging train stats at iteration 603-------------------- +[Apr 27, 2026 - 23:13:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_total_loss: 0.0141 iter count: 1.0 +[Apr 27, 2026 - 23:13:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_fake_score_loss: 0.0141 iter count: 1.0 +[Apr 27, 2026 - 23:13:20 | INFO | fastgen.callbacks.wandb:get_stat:335 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 27, 2026 - 23:13:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:661 ] WandB logging complete after 0.00 seconds diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/requirements.txt b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..a7117b8203d3b4df2b52c765d49a822a97b57cd9 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/requirements.txt @@ -0,0 +1,247 @@ +nvitop==1.6.1 +ftfy==6.3.1 +braceexpand==0.1.7 +antlr4-python3-runtime==4.9.3 +webdataset==1.0.2 +sentry-sdk==2.53.0 +rdkit==2025.9.5 +python-dotenv==1.2.1 +proglog==0.1.12 +omegaconf==2.3.0 +narwhals==2.17.0 +loguru==0.7.3 +imageio-ffmpeg==0.6.0 +plotly==6.5.2 +moviepy==2.2.1 +hydra-core==1.3.2 +toml==0.10.2 +wandb==0.25.0 +types-certifi==2021.10.8.3 +types-toml==0.10.8.20260408 +synchronicity==0.12.2 +propcache==0.4.1 +multidict==6.7.1 +hyperframe==6.1.0 +hpack==4.1.0 +frozenlist==1.8.0 +cbor2==5.9.0 +aiohappyeyeballs==2.6.1 +yarl==1.23.0 +watchfiles==1.1.1 +h2==4.3.0 +aiosignal==1.4.0 +grpclib==0.4.9 +aiohttp==3.13.5 +modal==1.4.1 +packaging==25.0 +setuptools==80.10.2 +wheel==0.46.3 +pip==26.0.1 +webencodings==0.5.1 +pure_eval==0.2.3 +ptyprocess==0.7.0 +nvidia-ml-py==13.590.48 +nvidia-cusparselt-cu12==0.7.1 +mpmath==1.3.0 +fastjsonschema==2.21.2 +zipp==3.23.0 +xyzservices==2025.11.0 +widgetsnbextension==4.0.15 +websocket-client==1.9.0 +webcolors==25.10.0 +wcwidth==0.6.0 +urllib3==2.6.3 +uri-template==1.3.0 +tzdata==2025.3 +typing_extensions==4.15.0 +triton==3.6.0 +traitlets==5.14.3 +tqdm==4.67.3 +tornado==6.5.5 +tinycss2==1.4.0 +sympy==1.14.0 +soupsieve==2.8.3 +smmap==5.0.3 +six==1.16.0 +sentencepiece==0.2.1 +Send2Trash==2.1.0 +safetensors==0.7.0 +rpds-py==0.30.0 +rfc3986-validator==0.1.1 +regex==2026.2.28 +pyzmq==27.1.0 +PyYAML==6.0.3 +python-json-logger==4.0.0 +Pygments==2.19.2 +pycparser==3.0 +psutil==7.2.2 +protobuf==4.24.4 +prometheus_client==0.24.1 +platformdirs==4.9.4 +pillow==11.3.0 +pexpect==4.9.0 +parso==0.8.6 +pandocfilters==1.5.1 +nvidia-nvtx-cu12==12.8.90 +nvidia-nvshmem-cu12==3.4.5 +nvidia-nvjitlink-cu12==12.8.93 +nvidia-nccl-cu12==2.27.5 +nvidia-curand-cu12==10.3.9.90 +nvidia-cufile-cu12==1.13.1.3 +nvidia-cuda-runtime-cu12==12.8.90 +nvidia-cuda-nvrtc-cu12==12.8.93 +nvidia-cuda-cupti-cu12==12.8.90 +nvidia-cublas-cu12==12.8.4.1 +numpy==1.26.4 +networkx==3.6.1 +nest-asyncio==1.6.0 +mistune==3.2.0 +MarkupSafe==3.0.3 +lark==1.3.1 +jupyterlab_widgets==3.0.16 +jupyterlab_pygments==0.3.0 +jsonpointer==3.0.0 +json5==0.13.0 +jmespath==1.1.0 +idna==3.11 +hf-xet==1.4.2 +h11==0.16.0 +fsspec==2026.2.0 +fqdn==1.5.1 +filelock==3.25.2 +executing==2.2.1 +einops==0.8.2 +defusedxml==0.7.1 +decorator==5.2.1 +debugpy==1.8.20 +cuda-pathfinder==1.4.2 +comm==0.2.3 +charset-normalizer==3.4.5 +certifi==2026.2.25 +bleach==6.3.0 +babel==2.18.0 +av==17.0.0 +attrs==25.4.0 +async-lru==2.2.0 +asttokens==3.0.1 +annotated-types==0.7.0 +typing-inspection==0.4.2 +terminado==0.18.1 +stack-data==0.6.3 +scipy==1.17.1 +rfc3987-syntax==1.1.0 +rfc3339-validator==0.1.4 +requests==2.32.5 +referencing==0.37.0 +python-dateutil==2.9.0.post0 +pydantic_core==2.41.5 +prompt_toolkit==3.0.52 +opencv-python-headless==4.11.0.86 +nvidia-cusparse-cu12==12.5.8.93 +nvidia-cufft-cu12==11.3.3.83 +nvidia-cudnn-cu12==9.10.2.21 +matplotlib-inline==0.2.1 +jupyter_core==5.9.1 +Jinja2==3.1.6 +jedi==0.19.2 +ipython_pygments_lexers==1.1.1 +importlib_metadata==8.7.1 +ImageIO==2.37.3 +httpcore==1.0.9 +gitdb==4.0.12 +cuda-bindings==12.9.4 +contourpy==1.3.3 +cffi==2.0.0 +beautifulsoup4==4.14.3 +anyio==4.12.1 +soundfile==0.13.1 +pydantic==2.12.5 +nvidia-cusolver-cu12==11.7.3.90 +jupyter_server_terminals==0.5.4 +jupyter_client==8.8.0 +jsonschema-specifications==2025.9.1 +ipython==9.11.0 +httpx==0.28.1 +GitPython==3.1.46 +botocore==1.42.68 +bokeh==3.9.0 +arrow==1.4.0 +argon2-cffi-bindings==25.1.0 +torch==2.10.0 +s3transfer==0.16.0 +jsonschema==4.26.0 +isoduration==20.11.0 +ipywidgets==8.1.8 +ipykernel==7.2.0 +argon2-cffi==25.1.0 +torchvision==0.25.0 +nbformat==5.10.4 +jupyter-console==6.6.3 +boto3==1.42.68 +accelerate==1.13.0 +nbclient==0.10.4 +jupyter-events==0.12.0 +nbconvert==7.17.0 +jupyter_server==2.17.0 +notebook_shim==0.2.4 +jupyterlab_server==2.28.0 +jupyter-lsp==2.3.0 +jupyterlab==4.5.6 +notebook==7.5.5 +jupyter==1.1.1 +fastgen==0.1.0 +pandas==3.0.1 +shellingham==1.5.4 +mdurl==0.1.2 +annotated-doc==0.0.4 +markdown-it-py==4.0.0 +rich==14.3.3 +typer==0.24.1 +huggingface_hub==1.7.1 +timm==1.0.25 +tokenizers==0.22.2 +diffusers==0.37.0 +transformers==5.3.0 +peft==0.18.1 +easydict==1.13 +lmdb==2.2.0 +threadpoolctl==3.6.0 +soxr==1.0.0 +msgpack==1.1.2 +llvmlite==0.47.0 +lazy-loader==0.5 +joblib==1.5.3 +audioread==3.1.0 +scikit-learn==1.8.0 +pooch==1.9.0 +numba==0.65.0 +librosa==0.11.0 +simsimd==6.5.16 +flatbuffers==25.12.19 +tifffile==2026.3.3 +stringzilla==4.6.0 +pyparsing==3.3.2 +prettytable==3.17.0 +onnx==1.17.0 +kiwisolver==1.5.0 +fonttools==4.62.1 +Cython==3.2.4 +cycler==0.12.1 +scikit-image==0.26.0 +onnxruntime==1.24.4 +matplotlib==3.10.8 +albucore==0.0.24 +albumentations==2.0.8 +insightface==0.7.3 +kornia_rs==0.1.10 +kornia==0.8.2 +pluggy==1.6.0 +iniconfig==2.3.0 +pytest==8.1.1 +dlib==20.0.1 +lpips==0.1.4 +python_speech_features==0.6 +decord==0.6.0 +click==8.2.1 +scenedetect==0.6.7.1 +torchaudio==2.10.0 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/wandb-metadata.json b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..4f1da843f52dd1c6eee1283dc30c468c621bbea9 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/files/wandb-metadata.json @@ -0,0 +1,72 @@ +{ + "os": "Linux-5.15.0-151-generic-x86_64-with-glibc2.39", + "python": "CPython 3.12.12", + "startedAt": "2026-04-27T01:02:45.745701Z", + "args": [ + "--config=fastgen/configs/experiments/OmniAvatar/config_sf_sink1_window7_redmd_beta2_taew.py", + "-", + "trainer.resume=False", + "model.fake_score_optimizer.lr=3e-6", + "log_config.group=omniavatar_sf_audiofix", + "log_config.name=sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769", + "log_config.project=OmniAvatar-FastGen", + "log_config.wandb_entity=paulhcho", + "model.sample_t_cfg.t_list=[0.999,0.769,0.0]" + ], + "program": "/home/work/.local/hyunbin/FastGen-redmd/train.py", + "codePath": "train.py", + "codePathLocal": "train.py", + "git": { + "remote": "https://paulcho98:@github.com/paulcho98/FastGen.git", + "commit": "76340fce27dac8c70c508cde190342d59ec341b7" + }, + "email": "paul.hyunbin@gmail.com", + "root": "/tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769", + "host": "main1", + "executable": "/home/work/.local/miniconda3/envs/hb_fastgen/bin/python3.12", + "cpu_count": 112, + "cpu_count_logical": 224, + "gpu": "NVIDIA H200", + "gpu_count": 4, + "disk": { + "/": { + "total": "1356758433792", + "used": "985738301440" + } + }, + "memory": { + "total": "2163961778176" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-4685d4b3-5cf9-2766-43d3-b9615a684b7c" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ec888a66-4b6f-b8de-b34b-249efb9ad262" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-9c1e1773-d710-06c9-7db7-1b54e9fc3790" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2b1017dc-2958-a946-16d2-2c29da6d18b0" + } + ], + "cudaVersion": "12.9", + "writerId": "ylk3419urzo1fyyskpspeagspxjgd264" +} \ No newline at end of file diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/logs/debug-core.log b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..900a7f677c734b4754bf82caa6737321cdaa4571 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/logs/debug-core.log @@ -0,0 +1,8 @@ +{"time":"2026-04-27T10:02:46.111350521+09:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp6udwwn4b/port-1078060.txt","pid":1078060,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-04-27T10:02:46.112228544+09:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1078060-1094024-2429011362/socket","Net":"unix"}} +{"time":"2026-04-27T10:02:46.112682435+09:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":1078060} +{"time":"2026-04-27T10:02:46.216226111+09:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-04-27T10:02:46.235475838+09:00","level":"INFO","msg":"handleInformInit: received","streamId":"yihddx5h","id":"1(@)"} +{"time":"2026-04-27T10:02:46.735092228+09:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"yihddx5h","id":"1(@)"} +{"time":"2026-04-27T10:02:52.746205766+09:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"k06p8qlfolku"} +{"time":"2026-04-27T23:14:05.618554866+09:00","level":"INFO","msg":"server: parent process exited, terminating service process"} diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/logs/debug-internal.log b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..d176778b9d99660ce81e2183c6c8f0534344eb33 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/logs/debug-internal.log @@ -0,0 +1,6 @@ +{"time":"2026-04-27T10:02:46.236048851+09:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-04-27T10:02:46.734345704+09:00","level":"INFO","msg":"stream: created new stream","id":"yihddx5h"} +{"time":"2026-04-27T10:02:46.73501439+09:00","level":"INFO","msg":"handler: started","stream_id":"yihddx5h"} +{"time":"2026-04-27T10:02:46.735085357+09:00","level":"INFO","msg":"stream: started","id":"yihddx5h"} +{"time":"2026-04-27T10:02:46.735100956+09:00","level":"INFO","msg":"sender: started","stream_id":"yihddx5h"} +{"time":"2026-04-27T10:02:46.735126398+09:00","level":"INFO","msg":"writer: started","stream_id":"yihddx5h"} diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/logs/debug.log b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..82089e6d2aa3595f58dcf5af0e4e2ca3a4a87edb --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/logs/debug.log @@ -0,0 +1,19 @@ +2026-04-27 10:02:45,746 INFO MainThread:1078060 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-04-27 10:02:45,746 INFO MainThread:1078060 [wandb_setup.py:_flush():81] Configure stats pid to 1078060 +2026-04-27 10:02:45,747 INFO MainThread:1078060 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-04-27 10:02:45,747 INFO MainThread:1078060 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/logs/debug.log +2026-04-27 10:02:45,747 INFO MainThread:1078060 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/logs/debug-internal.log +2026-04-27 10:02:45,747 INFO MainThread:1078060 [wandb_init.py:init():844] calling init triggers +2026-04-27 10:02:45,747 INFO MainThread:1078060 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'log_config': {'project': 'OmniAvatar-FastGen', 'group': 'omniavatar_sf_audiofix', 'name': 'sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769', 'wandb_mode': 'online', 'wandb_entity': 'paulhcho', 'wandb_credential': './credentials/wandb_api.txt'}, 'trainer': {'cudnn': {'deterministic': 'False', 'benchmark': 'True'}, 'checkpointer': {'save_dir': '/tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/checkpoints', 'use_s3': 'False', 's3_container': 's3://checkpoints/fastgen', 's3_credential': './credentials/s3.json', 'pretrained_ckpt_path': '/home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_syncnet_trained_shift_5_t769_4gpu_bs16_lr1e5_5000iter/checkpoints/0005000.pth', 'pretrained_ckpt_key_map': {'net': 'net'}}, 'callbacks': {'grad_clip': {'grad_norm': '10.0', 'model_key': 'net', '_target_': ""}, 'gpu_stats': {'every_n': '100', '_target_': ""}, 'train_profiler': {'every_n': '100', '_target_': ""}, 'param_count': {'_target_': ""}, 'ema': {'type': 'constant', 'beta': '0.9999', 'gamma': '16.97', 'ema_halflife_kimg': '500', 'ema_rampup_ratio': '0.05', 'start_iter': '0', '_target_': ""}, 'wandb': {'sample_logging_iter': '100', '_target_': "", 'fps': '25', 'syncnet_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model'}}, 'save_ckpt_iter': '100', 'validation_iter': '100', 'skip_initial_validation': 'True', 'logging_iter': '1', 'max_iter': '5000', 'visualize_teacher': 'False', 'seed': '0', 'val_seed': None, 'resume': 'False', 'ddp': 'False', 'fsdp': 'True', 'tf32_enabled': 'True', 'grad_accum_rounds': '2', 'batch_size_global': None, 'offload_module_in_decoding': 'False', 'fsdp_cpu_offload': 'False', 'fsdp_min_num_params': '10000000', 'fsdp_sharding_group_size': None, 'global_vars': None, 'global_vars_val': [None], 'augment_pipe': None}, 'dataloader_train': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_path.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '8', 'num_workers': '2', 'neg_text_emb_path': None, 'use_ref_sequence': 'True', '_target_': "", 'load_raw_audio': 'True', 'raw_audio_sample_rate': '16000', 'raw_audio_num_frames': '81', 'raw_audio_fps': '25.0'}, 'dataloader_val': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_val10.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '1', 'num_workers': '2', 'neg_text_emb_path': None, 'use_ref_sequence': 'True', 'load_ode_path': 'False', '_target_': ''}, 'eval': {'num_samples': '50000', 'save_images': 'False', 'min_ckpt': '0', 'max_ckpt': '100000000', 'samples_dir': 'samples'}, 'model': {'net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'chunk_size': '3', 'total_num_frames': '21', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt', 'net_pred_type': 'flow', 'schedule_type': 'rf', 'local_attn_size': '7', 'sink_size': '1', 'use_dynamic_rope': 'True', '_target_': ""}, 'teacher': {'model_size': '14B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_maskall_refseq_mouth_weight_4gpu/step-6000.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'fake_score_net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_maskall_refseq_mouth_weight_2gpu/step-1000.pt', 'merge_lora': 'False', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'guidance_scale': '4.5', 'timestep_cfg': {'enabled': 'True', 't_lo': '0.556', 't_hi': '0.882'}, 'skip_layers': None, 'net_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'net_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'sample_t_cfg': {'time_dist_type': 'shifted', 'train_p_mean': '-1.1', 'train_p_std': '2.0', 'shift': '5.0', 'min_t': '0.001', 'max_t': '0.999', 't_list': ['0.999', '0.769', '0.0'], 'log_t_df': '0.01'}, 'input_shape': ['16', '21', '64', '64'], 'device': 'cuda', 'grad_scaler_enabled': 'False', 'grad_scaler_init_scale': '65536.0', 'grad_scaler_growth_interval': '2000', 'pretrained_model_path': '', 'pretrained_student_net_path': '', 'load_student_weights': 'False', 'enable_preprocessors': 'True', 'use_ema': 'False', 'student_sample_steps': '2', 'student_sample_type': 'sde', 'fsdp_meta_init': 'False', 'add_teacher_to_fsdp_dict': 'True', 'ddp_find_unused_parameters': 'True', 'precision': 'bfloat16', 'precision_amp': None, 'precision_amp_infer': None, 'precision_amp_enc': None, 'precision_fsdp': 'float32', 'fake_score_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '3e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'fake_score_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'discriminator': {'feature_indices': '{0, 1, 2}', 'all_res': ['32', '16', '8'], 'in_channels': '256', '_target_': ""}, 'discriminator_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '0.0001', 'weight_decay': '0.01', 'betas': ['0.9', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'discriminator_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'student_update_freq': '5', 'gan_loss_weight_gen': '0', 'gan_use_same_t_noise': 'False', 'fake_score_pred_type': 'x0', 'gan_r1_reg_weight': '0.0', 'gan_r1_reg_alpha': '0.1', 'enable_gradient_in_rollout': 'True', 'start_gradient_frame': '0', 'same_step_across_blocks': 'True', 'last_step_only': 'False', 'context_noise': '0.0', 'fake_score': None, 'reward': {'enabled': 'True', 'checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model', 'input_fps': '25.0', 'audio_sample_rate': '16000', 'vshift': '15', 'decoder_kind': 'taew', 'taew_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth'}, 'reward_beta': '2', 'center_reward': 'False', 'clamp_reward': None, 'reward_weighting_mode': 'per_sample', 'save_reward_debug_video': 'False', 'reward_debug_dir': 'logs/redmd_debug'}, 'model_class': {'config': None, '_target_': ""}, '_wandb': {}} +2026-04-27 10:02:45,747 INFO MainThread:1078060 [wandb_init.py:init():892] starting backend +2026-04-27 10:02:46,216 INFO MainThread:1078060 [wandb_init.py:init():895] sending inform_init request +2026-04-27 10:02:46,231 INFO MainThread:1078060 [wandb_init.py:init():903] backend started and connected +2026-04-27 10:02:46,235 INFO MainThread:1078060 [wandb_init.py:init():973] updated telemetry +2026-04-27 10:02:46,265 INFO MainThread:1078060 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-04-27 10:02:47,391 INFO MainThread:1078060 [wandb_init.py:init():1042] starting run threads in backend +2026-04-27 10:02:47,739 INFO MainThread:1078060 [wandb_run.py:_console_start():2524] atexit reg +2026-04-27 10:02:47,739 INFO MainThread:1078060 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-04-27 10:02:47,739 INFO MainThread:1078060 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-04-27 10:02:47,739 INFO MainThread:1078060 [wandb_run.py:_redirect():2465] Redirects installed. +2026-04-27 10:02:47,746 INFO MainThread:1078060 [wandb_init.py:init():1082] run started, returning control to user process diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/run-yihddx5h.wandb b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/run-yihddx5h.wandb new file mode 100644 index 0000000000000000000000000000000000000000..5db41162ea0d569216ec22bd23247ff242b1732a --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb/run-20260427_100245-yihddx5h/run-yihddx5h.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06041fd046f7022b3b4ee1a07e9603219cc0b8627b90843d247b512429fa2734 +size 10682368 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb_id.txt b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb_id.txt new file mode 100644 index 0000000000000000000000000000000000000000..42c19c20af1a0ddd9c90916466231ed4c7cdfe67 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_SYNCNET_MOUTHWEIGHT_FSMATCHED_T769/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_syncnet_mouthweight_fsmatched_t769/wandb_id.txt @@ -0,0 +1 @@ +yihddx5h