diff --git a/.gitattributes b/.gitattributes index d2469f62e1991121dd2a34cbfc14a4fe8ac7c1b9..c1c6d515e0e1ff1e60889a487ddb22516e0c1ecd 100644 --- a/.gitattributes +++ b/.gitattributes @@ -289,3 +289,206 @@ output_omniavatar_v2v_1.3B_maskall/wandb/run-20260306_052921-tq0ucjf3/files/medi output_omniavatar_v2v_1.3B_maskall/wandb/run-20260306_052921-tq0ucjf3/files/media/videos/val_recon/videos_500_c1ffabc716a145588cec.mp4 filter=lfs diff=lfs merge=lfs -text output_omniavatar_v2v_1.3B_maskall/wandb/run-20260306_052921-tq0ucjf3/files/media/videos/val_recon/videos_500_d15481d6e46ce24778b2.mp4 filter=lfs diff=lfs merge=lfs -text output_omniavatar_v2v_1.3B_maskall/wandb/run-20260306_052921-tq0ucjf3/run-tq0ucjf3.wandb filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000100.fake_score_model/.metadata filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000100.fake_score_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000100.fake_score_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000100.fake_score_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000100.fake_score_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000100.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000100.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000100.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000100.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000100.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000200.fake_score_model/.metadata filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000200.fake_score_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000200.fake_score_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000200.fake_score_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000200.fake_score_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000200.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000200.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000200.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000200.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000200.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000300.fake_score_model/.metadata filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000300.fake_score_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000300.fake_score_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000300.fake_score_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000300.fake_score_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000300.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000300.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000300.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000300.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000300.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000400.fake_score_model/.metadata filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000400.fake_score_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000400.fake_score_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000400.fake_score_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000400.fake_score_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000400.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000400.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000400.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000400.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000400.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000500.fake_score_model/.metadata filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000500.fake_score_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000500.fake_score_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000500.fake_score_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000500.fake_score_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000500.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000500.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000500.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000500.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000500.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.fake_score_model/.metadata filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.fake_score_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.fake_score_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.fake_score_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.fake_score_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.fake_score_optim/.metadata filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.fake_score_optim/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.fake_score_optim/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.fake_score_optim/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.fake_score_optim/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.net_model/.metadata filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.net_model/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.net_model/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.net_model/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.net_model/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.net_optim/.metadata filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.net_optim/__0_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.net_optim/__1_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.net_optim/__2_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.net_optim/__3_0.distcp filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/train_media/data/real_100_47403c800591fce4a99b.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/train_media/data/real_200_4e05a85e239f00a00e38.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/train_media/data/real_300_97779d11e00d4fea84a0.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/train_media/data/real_400_112c0905448a821974f0.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/train_media/data/real_500_2594e32c120a620ede9d.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/train_media/data/real_600_aebf0f3e2314a893fcf4.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/train_media/student/generation_100_dbed78ee0fdec1d90f2e.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/train_media/student/generation_200_f1ca6fb9f7ed2cf17ff3.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/train_media/student/generation_300_ccd008653d34c4e1079d.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/train_media/student/generation_400_607d6362f8aa8d1a9233.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/train_media/student/generation_500_1338a835b20737523158.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/train_media/student/generation_600_f8fe5f5817f741e0fa2e.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_100_065388816206e3899e69.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_100_22be8aa64c3275adc278.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_100_629f5e819c61eb78324e.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_100_6906457b94f1950af020.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_100_80ab16039c803c1187c0.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_100_83109bf9fb68148a0b72.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_100_882bdb3096251d76a010.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_100_9593187e51a6d7571ed2.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_100_a35a541a84c35b1c8ca4.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_100_d438e3becfef4d908558.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_200_08b05745233fe6193330.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_200_473440a7b653e2454a25.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_200_6fdcaae10a79aa2872ec.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_200_7dcf75e9215bb38e7093.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_200_7f4db1587fc269fd3729.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_200_a455d874652f89da8df7.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_200_abfcbdfc4ceabd160d2f.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_200_b0cf3c464e74a54def67.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_200_b836dd841434b50dbb1e.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_200_d280cac5fa1c9a11802d.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_300_0df52ffc1f51a11a274f.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_300_1fb9504f0da3470e5499.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_300_5ad6839552a5c758173d.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_300_6d315d601d30bcf4f5ff.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_300_9ff25f0e613106da8677.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_300_a6ff81de78f80198f4e8.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_300_bb5e13235464274ec0a4.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_300_cef4b9d308aebb76139c.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_300_d4dabdf6ff31d17a324a.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_300_d86650f7bc1e730c3ee1.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_400_079a3388458485b0ce5f.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_400_272ad7a28e6015af5078.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_400_470a673634ca0778a523.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_400_5d7d48636aef730f2fef.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_400_671e25a6f515483157b7.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_400_7bea112416b01940b4f4.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_400_7f84241ae237b80cf500.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_400_d0dc14a28c02520d2844.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_400_d23a2ae68b13caed8e64.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_400_f3ed3833ea95e0d907b5.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_500_0d712ca5c3286b44ef10.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_500_157dfe635f0203c31db0.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_500_47537149d882f627a6c9.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_500_561404fbd13b0dc9f796.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_500_7e7a3a035a1c7faf3b7c.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_500_7f1d9130a8ce38f683f1.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_500_80b5402cd70bbb987599.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_500_99ba44fb1c4871734865.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_500_de98e19993c9db197ada.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_500_ec6f3115c2698671e276.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_600_117f332517d46edf01fd.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_600_1db9359d35ea2edc77ef.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_600_3b016fb2a032e42c4746.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_600_7027ebe09b83c2128696.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_600_714d74f6d5a878a7b6db.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_600_76db7dc840b20b5fc96c.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_600_877c5e326d3373357f9f.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_600_be7cb5d2d76a5b1db3ef.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_600_fcf497d71dd93a3aefa9.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_600_fd09907d49b2ad927921.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_100_15ac5b765a401d035db1.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_100_294040ac378ceaa34329.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_100_6ff046418b5d41477989.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_100_80aab00fcfa85d5db98e.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_100_884b2d197a5511393680.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_100_a076d17e202ed18ebdf8.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_100_c0196720a45889bb2198.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_100_c0f892b5fba78348af85.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_100_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_100_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_200_15ac5b765a401d035db1.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_200_294040ac378ceaa34329.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_200_6ff046418b5d41477989.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_200_80aab00fcfa85d5db98e.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_200_884b2d197a5511393680.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_200_a076d17e202ed18ebdf8.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_200_c0196720a45889bb2198.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_200_c0f892b5fba78348af85.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_200_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_200_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_300_15ac5b765a401d035db1.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_300_294040ac378ceaa34329.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_300_6ff046418b5d41477989.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_300_80aab00fcfa85d5db98e.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_300_884b2d197a5511393680.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_300_a076d17e202ed18ebdf8.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_300_c0196720a45889bb2198.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_300_c0f892b5fba78348af85.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_300_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_300_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_400_15ac5b765a401d035db1.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_400_294040ac378ceaa34329.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_400_6ff046418b5d41477989.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_400_80aab00fcfa85d5db98e.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_400_884b2d197a5511393680.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_400_a076d17e202ed18ebdf8.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_400_c0196720a45889bb2198.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_400_c0f892b5fba78348af85.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_400_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_400_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_500_15ac5b765a401d035db1.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_500_294040ac378ceaa34329.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_500_6ff046418b5d41477989.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_500_80aab00fcfa85d5db98e.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_500_884b2d197a5511393680.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_500_a076d17e202ed18ebdf8.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_500_c0196720a45889bb2198.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_500_c0f892b5fba78348af85.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_500_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_500_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_600_15ac5b765a401d035db1.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_600_294040ac378ceaa34329.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_600_6ff046418b5d41477989.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_600_80aab00fcfa85d5db98e.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_600_884b2d197a5511393680.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_600_a076d17e202ed18ebdf8.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_600_c0196720a45889bb2198.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_600_c0f892b5fba78348af85.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_600_f5463a4ea6f68a39c5d0.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_600_f7412c9e170ffd299748.mp4 filter=lfs diff=lfs merge=lfs -text +FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/run-fxpzd9v7.wandb filter=lfs diff=lfs merge=lfs -text diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000100.fake_score_model/.metadata b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000100.fake_score_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..110436242e749cdb9ac7737e8844283bcbbbf2eb --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000100.fake_score_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99bb1896b03c29f2d517be85dbfe28a3fc7235d586c4d7ef2af89ddf004b1ce5 +size 1191937 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000100.fake_score_model/__0_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000100.fake_score_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..7b06e6f71dbe68a1b760c053ba051a48f57facff --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000100.fake_score_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd1842e553ccac782b08f7f2b1ad95fdccafd27a389dd39f16ebc40fc39c1a29 +size 1599501209 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000100.fake_score_model/__1_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000100.fake_score_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..b61d5e3ced81604cc972bee959227f0f0f8d73c9 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000100.fake_score_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d8264a4f4fb01562021a759a9052cee49926481ecfd3911a007253535b186f8 +size 1598334114 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000100.fake_score_model/__2_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000100.fake_score_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..b466ee099fbd3e12ecff45853a32edee8a84018c --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000100.fake_score_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d85b7de7efb83a3afa5de634088f8cc5a9272a880d7207dc1008763e1f33238 +size 1598358322 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000100.fake_score_model/__3_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000100.fake_score_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..b3e63d9b997c2be36c92e90e2cdcd47e18203df3 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000100.fake_score_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26a8233c3de8de1946be6a3bffa7aef247c49f03e911a476a70f3b0c08448d26 +size 1598356809 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000100.net_model/.metadata b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000100.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..1a6d30820138f37ae3a7aeb5c078dee47bc8140a --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000100.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43505688150e34f09008a6347d6c9c6c551aa1ac0f5775a6091118e390c998e4 +size 614374 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000100.net_model/__0_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000100.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..208faea2455eb762803870d5d98586fc0a84b958 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000100.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26ba5988e607ad0286ea8572b3f95ee5996abe48b123d5e50065a190ccefff98 +size 1424446005 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000100.net_model/__1_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000100.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..7adaa6cc588e66fab78699530ee29f10e7e82504 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000100.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42f0e6cd04dd7b921b0839dcfc7f1671c97a6ac0a8b0825730d0c992fad32716 +size 1401881227 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000100.net_model/__2_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000100.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..f4fb5ce126a33088fa8436f02c6e556023492634 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000100.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71e4b68bd54f18d359b14dad20e7caabef89abc5b385707c76f59d07fbe54cc4 +size 1406759189 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000100.net_model/__3_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000100.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..74c64074f2955a6ab9579e470cf484ec62c4d89f --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000100.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cce58499242f1e5a0801f12e61a212ce49b570b1ab268ff504cd813d02255dd3 +size 1401210558 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000100.pth b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000100.pth new file mode 100644 index 0000000000000000000000000000000000000000..0f637e0f6792d8ae1960e0e448573321eacbddc4 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000100.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c9e7bf7b483afea9b5f5fc9f6d1a8368eea2246e5e5404c08b9fba8acdc0064 +size 1901 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000200.fake_score_model/.metadata b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000200.fake_score_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..88f6c8ac1f1f48f8beb9f8b1895f2fd88df8a103 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000200.fake_score_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a7d896e85f87502a9fa0d464b9863d0944379e644c5b4064035710227e2157e +size 1191937 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000200.fake_score_model/__0_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000200.fake_score_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..f78c4dae27352b3f258a2a0505ccddc84af5f81a --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000200.fake_score_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0725d013cf201e1887e513bd2f515cab3979af20f6f3edc46d125ae5f3b29bc7 +size 1599501209 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000200.fake_score_model/__1_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000200.fake_score_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..be503db73904e23749d3aac82dd459e6a6463819 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000200.fake_score_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:affe275e9178a7a53fa578584ef2d1c50f9dc0e4b762e363c91a09b576846ca3 +size 1598334114 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000200.fake_score_model/__2_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000200.fake_score_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..d1de42d7c6ecad66f3b16ff70fa8fe97922e49ad --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000200.fake_score_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1e4fa009399f81dd99a05f8d61f43ace36f52b20c3fbfaa281892a4c7fb947a +size 1598358322 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000200.fake_score_model/__3_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000200.fake_score_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..faaf2464b76aaea0705a00a1cd4259844e109281 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000200.fake_score_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c1bce01c402e2855bdeb5659939352554d7ce3342134186d5c71c425005c5a6 +size 1598356809 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000200.net_model/.metadata b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000200.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..8de46012cb523262b766c134b913cf0e30f4b160 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000200.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ae3963b1833baebeaa358546b7da72456196b1951a98c5a5dfefd211c1aa036 +size 614374 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000200.net_model/__0_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000200.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..701a9ca3dc77d532680ef095cc2bb21ac13853f0 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000200.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc74c4b00bf81f2ca41e7546cdafcc32671bf22486eaf54a1b92dfb75d1b6540 +size 1424446005 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000200.net_model/__1_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000200.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..e14dc6193e38789611889c6fcf55271795c84d52 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000200.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18f213506515f85e19fb2a2baf3fa6c0d0207b0d5efada0788d2f532e3c8f8af +size 1401881227 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000200.net_model/__2_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000200.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..cde0b8fadd9e173f7227b17e028fc6c78f079628 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000200.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b06713b9592ebedcdebbb2100a837290c53e8db3ad686b5724f5e45b415ffb31 +size 1406759189 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000200.net_model/__3_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000200.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..f70c1162537cb6169f885d4c6d7e19bf49c1b919 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000200.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:449d2e9d4b08f14959e1e9dc93ba4fd3dae43dbd892bf83c36685deee7cd13ec +size 1401210558 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000200.pth b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000200.pth new file mode 100644 index 0000000000000000000000000000000000000000..94689c6d88777305ec14be3e82dcf676c28c81a1 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000200.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18d49c29dddf4cff9abe84c331dfad17799824bf4158c145f77812c5430228f1 +size 1901 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000300.fake_score_model/.metadata b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000300.fake_score_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..39f4e807c5ef4b54d8438db035e8961b7d7e6e79 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000300.fake_score_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66a6023bba639c008dd8e165ffa0c6f0e14c01b6d5276629f1fadeb4607c6e05 +size 1191937 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000300.fake_score_model/__0_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000300.fake_score_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..de63603906cae1b16630a460d207548479a8c9dd --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000300.fake_score_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d57e9c319a070a901a972d6d1d8d0425c5bc6951ae086b06ec90befd8f368947 +size 1599501209 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000300.fake_score_model/__1_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000300.fake_score_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..0bfaeda82b88775befce0029fb1d0faa8efc657d --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000300.fake_score_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5cf2f4235ac23d10830147b6d5c55e2713b9129a12a71890251aef9a4742104a +size 1598334114 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000300.fake_score_model/__2_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000300.fake_score_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..30bf559798135d670512b723c8533badd783611d --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000300.fake_score_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:273554fc2d8483c34dd55f9275ec96de66758fe81b7ba050541a50d4195823ce +size 1598358322 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000300.fake_score_model/__3_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000300.fake_score_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..c895ad18f5b006001d935df4c74a443b81b478c5 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000300.fake_score_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b1ba187baa3eb1fa0eb9265a9630faecfc5a608f3927c8a0bc5008fc5c3b78a6 +size 1598356809 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000300.net_model/.metadata b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000300.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..0799fc8477cab811e5ab478ca5fae4b0d41a6e4f --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000300.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5c7e10e7d909df8b94fd0f5f0733fff69ce5ddfac569f8a6fc912e55804bf23 +size 614374 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000300.net_model/__0_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000300.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..241459942f32eb64edc916e266eb165a03a0dff0 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000300.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5aaf9e067bd4667bb5e9252078c3e9adee91563529426d9301ba77238c2a9d60 +size 1424446005 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000300.net_model/__1_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000300.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..9f394784b3c87a0dc9cf06c98a6e46c3f9f98d5d --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000300.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:624a581f10efed8a91d9ef91bf000aa051555ec0dcfda816bc4b127f428cc0cc +size 1401881227 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000300.net_model/__2_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000300.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..57aa033ca0fc5c14e49ce60f945b526f189cc3d9 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000300.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3f3e365ef761e3d1024c7a7c499e28dfb137c285c7bd7af8623c5cd71038528 +size 1406759189 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000300.net_model/__3_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000300.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..9e5d9e46c70a94493830a5c3575db28b2c450f28 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000300.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0631ded8fbd9f7f87d5478dccfaadd886d040984b555a241f2584ca7c8ca19b +size 1401210558 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000300.pth b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000300.pth new file mode 100644 index 0000000000000000000000000000000000000000..36cdfef63708c11e993e5164fd9859eb64d6d33e --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000300.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:463104a57d6bfde142e5cd01a059afdb8b99dad40c73e04a236502ee34fb4d66 +size 1901 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000400.fake_score_model/.metadata b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000400.fake_score_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..a3f6765496aa0660bf6d4756e875a2d503487220 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000400.fake_score_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3947743393a64ebeebf5e75b2bf2e4cdbb1295a49b1ebbc85f62fec2c4b36596 +size 1191937 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000400.fake_score_model/__0_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000400.fake_score_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..770af27adec1c23eab1385aa4707003293ec704e --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000400.fake_score_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b57ba62766ba6a71b2dd77e4f4b1d6f80a815b152d5db256aaca677bc133f629 +size 1599501209 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000400.fake_score_model/__1_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000400.fake_score_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..a7f4d8f3f3ce515eb1a437126c174b42dc8a6ba6 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000400.fake_score_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:989e07d275e2a7101e9677b8f3c34e54fafe9bb42676cb1ed68db851fea8f63f +size 1598334114 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000400.fake_score_model/__2_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000400.fake_score_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..206cfdc722fac13d7be0ada9b57fbe3d42eba8db --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000400.fake_score_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc51b4783a38d88a2752331a52607c966c7247cf7d9684cee8fcd76fedfda47b +size 1598358322 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000400.fake_score_model/__3_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000400.fake_score_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..270af6ba54eb87edb3c07e83ac937229b0301ad9 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000400.fake_score_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9b8e18a034bf171b4477d1ac12b5210f10c7e61563205230ab97bfdb0f0b8ec +size 1598356809 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000400.net_model/.metadata b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000400.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..6663d7c8c3d8e27ea21dd61f8079f9200635ff6b --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000400.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d9e6c3ae053f245c3b837c981150532c2dbb38f17059aa91666c693fa97cd81e +size 614374 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000400.net_model/__0_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000400.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..4cb70ce6a7212f431bc3203cadb16351ebe114d9 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000400.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f6375a756f17a1545e85880921c9213eb800d3165973c4952a94c66f6e5b719 +size 1424446005 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000400.net_model/__1_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000400.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..49a5e17ea18b8262f5ea133fab662041410f1b8b --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000400.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb79eba3f3498e11ca9571d6a80cb3c9865b0fa0528c183848ee2209cf950911 +size 1401881227 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000400.net_model/__2_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000400.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..8faaa33263128f2a7fed19ef09312aa6a995858a --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000400.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:276737bad804daf2a48de3ae3cda07ee043ad1d939f7567186ad9901ae6ad00d +size 1406759189 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000400.net_model/__3_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000400.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..c7547a0f75fff02a79e47924c464ab951f9349d6 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000400.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35097f49d9935be6d6633afdd344fa36037d0917470fe368fd1f02f2e7c18f35 +size 1401210558 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000400.pth b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000400.pth new file mode 100644 index 0000000000000000000000000000000000000000..0deb1499fd51197d9815f644a93381fbe0678161 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000400.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:31545bbbd2979dddf082789dbf999d9324dc9febf1b016304ddc713ede429c3e +size 1901 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000500.fake_score_model/.metadata b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000500.fake_score_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..12be15b0c2efc695b8d3211e39938b1ef099937d --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000500.fake_score_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bbf2a57f9e08903d78b433cf94cc04bce1bbcd698724e0f8b097dfbc9da6bbe0 +size 1191937 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000500.fake_score_model/__0_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000500.fake_score_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..3a7002d68e672e0fc2cd0f0e8dc9e7b35ea635c8 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000500.fake_score_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c84a6106a2f96ec8590f7b5be03a968fe0d19af433693cf15255bf103c47c50f +size 1599501209 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000500.fake_score_model/__1_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000500.fake_score_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..ef02f8c6fbebcae2d2c9e2a49e72298df0aa095b --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000500.fake_score_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67d4c9df626b5fdc513638374ccd4d29ee431043b215c08223a253d02b3744b2 +size 1598334114 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000500.fake_score_model/__2_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000500.fake_score_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..0c3cf004477b2b235efb4f10e1ddf9da236f696c --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000500.fake_score_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08a3b0c64d474d2581585b5a10c85cd196018ea3f8b82dcef0bd8242be388498 +size 1598358322 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000500.fake_score_model/__3_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000500.fake_score_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..7fbaf1c8ca307239bb5f7073ebb7cb22026cf3f1 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000500.fake_score_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d99f0c9a044ffbaf40bbf3856d134b11ecdd1d215bae5e7c8e898b3d783419c +size 1598356809 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000500.net_model/.metadata b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000500.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..1e80284f6d606ac48c313413388e71051e04450f --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000500.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e8163f1903b9dc100d83f51219455c6056b7c9a48c8396c08c1fbbdf35f4e9c +size 614374 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000500.net_model/__0_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000500.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..fa60f4d5cbe4a71bf1bf122baa4643cf4788f094 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000500.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d70f87e4ad15927e211ae7d47c1051ffabd92e73804a6ef9a5507d5c907a2fe +size 1424446005 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000500.net_model/__1_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000500.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..788f87b6675be16fa29e52081fddc3b6128c22fc --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000500.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3b92d7fcda2c43dbbb925ec411d8c27281b3a739f26d7cfd998812e64693ae6 +size 1401881227 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000500.net_model/__2_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000500.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..3436f49f89d0617f5fe2ee74ad6f540c0332b560 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000500.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26a7db82e1a471a41d1fbbad509efa1bdd2429b447eaf5ac53dbb705eb5cc783 +size 1406759189 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000500.net_model/__3_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000500.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..a38578beaad84b5eedc611545a8f03f5269abfc5 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000500.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d61cc5e831265e9df12fd309eee018918cf4a2abd173bc874f9f521a1ff2296b +size 1401210558 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000500.pth b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000500.pth new file mode 100644 index 0000000000000000000000000000000000000000..df0fb483027e40ac33d6a36f4b38fcfd244d6a44 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000500.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a2110994f1f90ec15f3f831251e70d29fe1b83f8a35bb969069c4ac3f211c6d +size 1901 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.fake_score_model/.metadata b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.fake_score_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..e12e927b18b9fdc8eff2f8c0d5bd7ec31aed55ef --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.fake_score_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0068bb149023d45bc49a97dabcfe397cce539da5e2bb0eae632cd058f66d6e5e +size 1191937 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.fake_score_model/__0_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.fake_score_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..62e41fd6703109c71940e56af951d00abd2205e5 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.fake_score_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7320e92f3c94aab135cd415a5a3d8eb71dcb5300713a833b1ed31ad6020a57bf +size 1599501209 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.fake_score_model/__1_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.fake_score_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..f45178031e375ea8eaea30a9c1c833591acdd399 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.fake_score_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a651721d8424901f1b8d8d6dacbcf49fe95cf6035b0ddf8947df7e1faef7bd32 +size 1598334114 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.fake_score_model/__2_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.fake_score_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..c84c122cd10f15cd1a6f83cb45e38c05c1b75cf3 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.fake_score_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a8d3b7230e67dc0abc5678f4b07f27c61f02e8cb4e2d7fab29bf14ed834683f +size 1598358322 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.fake_score_model/__3_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.fake_score_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..94233cfe2d4d6bd9a8479f4f4f5de123817a2db9 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.fake_score_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80e0d6bed9e520a28b24b4a52a824664bbf7429acd64e5e8126a96dab585fc68 +size 1598356809 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.fake_score_optim/.metadata b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.fake_score_optim/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..d74d43d6d8203dba3aa9c115a1b1f778731e29a6 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.fake_score_optim/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28d20eab80fb63c8319b3a8835a46ff7cf40db2dc12dd9302a7ee90cd6815f51 +size 1387221 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.fake_score_optim/__0_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.fake_score_optim/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..e26a9d3b6eace91c3e017cfb34d234b07278cf7c --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.fake_score_optim/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e80daf3fab050e2be28f4cc9f66ac57fbfa54b07d1879524d4070066e072daed +size 351856217 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.fake_score_optim/__1_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.fake_score_optim/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..288b23f82326fe3f0d51bce841095971341599d3 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.fake_score_optim/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9d2d48bfdd9dcb4c5f8c8f075a75dd76645294560a0cbf1bb8f5a14313ca346 +size 352173823 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.fake_score_optim/__2_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.fake_score_optim/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..94dbf4b3e48a024774141f620ef0815c5d091746 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.fake_score_optim/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e6c969db566283814fbf086ba5db91164ef9dc49dad9eae66df8c59905b3dc3 +size 352173823 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.fake_score_optim/__3_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.fake_score_optim/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..40b44c8eebed92daab43f6f82d6043800d397468 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.fake_score_optim/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce0b34ed07e1f62e0d8f26042ab19f2294b26453dc47a4e24bec2a332243ed2c +size 352211338 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.net_model/.metadata b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.net_model/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..cdf2dafae2f4e50d9457a319143a6ec0d2f6aaf8 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.net_model/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dcabf9ac14d33c54a5371b0ce5a61b3cfcd35af745a8e28e88e6a7c8e63334c8 +size 614374 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.net_model/__0_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.net_model/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..206e0f828fa3b4dbc482036c714c0ffd4ec14881 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.net_model/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0cf009eb95ea5915fd0c5d6077fd15290b63cc0bf9c4b7f397c22e7ede010511 +size 1424446005 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.net_model/__1_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.net_model/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..f6b40562f3b55195230aec64f0490a5c03a7c9d9 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.net_model/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03fa1a191d379d81d7f4b4ed37f74c27ec8f3c71645c6fe2641275b6781ab1dd +size 1401881227 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.net_model/__2_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.net_model/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..8d89a98b3331ba9d24ed5a2dd48230da57394397 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.net_model/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58657ddae95b24d5e6a1f423ced9830fab5ca18863ea5f98353bff1f336c72b7 +size 1406759189 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.net_model/__3_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.net_model/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..8c7ff5b9474b6499ffab5c0bd234a91596ae33c7 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.net_model/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ddc274c14319a68e6f9285d0207ec2ff410c3248041eb32cb8d1a6dc8eb5b8f +size 1401210558 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.net_optim/.metadata b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.net_optim/.metadata new file mode 100644 index 0000000000000000000000000000000000000000..6a013844a28bc801b352850bcc4b30239ec0d544 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.net_optim/.metadata @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06e4648e41a7be25b2fd5c06e5034ca55ee2c06361e8b2e0528da8e9696dacc1 +size 1649263 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.net_optim/__0_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.net_optim/__0_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..78433f26884061e1eacade4de86faea6716aa084 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.net_optim/__0_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd0b8cf6ab2276b04f35057eda1a9ed0b62369a5597fae0f99d550bfa253380e +size 2803269738 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.net_optim/__1_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.net_optim/__1_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..fb5fdd2bbf48cf7e9a42a8f0aaca3eec91d7bbfa --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.net_optim/__1_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ba1a2d44357297a71f6bf1e6e829c5563ce36abb36d8d90429fe65bcec0fd90 +size 2827716077 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.net_optim/__2_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.net_optim/__2_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..31f10c7ff0d1c27fb745d0aaeec6d72356288f2b --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.net_optim/__2_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a1f46c6ea3d4e89db9a733bfa2c727eb9fe8ebd2a855325a984b8236c5879d4 +size 2827716077 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.net_optim/__3_0.distcp b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.net_optim/__3_0.distcp new file mode 100644 index 0000000000000000000000000000000000000000..74f095534ba41600745cfca4e3f6f98f8b8ee4d7 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.net_optim/__3_0.distcp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82400d76d9b90b2c24461f9720ff85aa7aabbef0fba903a5fbede4e4b4f81cdd +size 2811295404 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.pth b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.pth new file mode 100644 index 0000000000000000000000000000000000000000..130c61950c21828925cda2c16a5be77c70be72e4 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c097cd2564e2f877acb5e00b0b08cbb2ce9c3d3500939503b1bf26d185a049e +size 1901 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/config.yaml b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9d2a435480457ecf8dc18d88a30aa1883e7c8b79 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/config.yaml @@ -0,0 +1,284 @@ +dataloader_train: + _target_: + batch_size: '8' + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_path.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_raw_audio: 'True' + neg_text_emb_path: null + num_workers: '2' + raw_audio_fps: '25.0' + raw_audio_num_frames: '81' + raw_audio_sample_rate: '16000' + use_ref_sequence: 'True' +dataloader_val: + _target_: + batch_size: '1' + data_list_path: /home/work/stableavatar_data/v2v_training_data/video_square_val10.txt + latentsync_mask_path: /home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png + load_ode_path: 'False' + neg_text_emb_path: null + num_workers: '2' + use_ref_sequence: 'True' +eval: + max_ckpt: '100000000' + min_ckpt: '0' + num_samples: '50000' + samples_dir: samples + save_images: 'False' +log_config: + group: omniavatar_sf_audiofix + name: sf_sink1_window7_redmd_audiofix_beta2_taew_4step + project: OmniAvatar-FastGen + wandb_credential: ./credentials/wandb_api.txt + wandb_entity: paulhcho + wandb_mode: online +model: + add_teacher_to_fsdp_dict: 'True' + center_reward: 'False' + clamp_reward: null + context_noise: '0.0' + ddp_find_unused_parameters: 'True' + device: cuda + discriminator: + _target_: + all_res: + - '32' + - '16' + - '8' + feature_indices: '{0, 1, 2}' + in_channels: '256' + discriminator_optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'False' + lr: '0.0001' + model: null + optim_type: adamw + weight_decay: '0.01' + discriminator_scheduler: + _target_: + cycle_lengths: + - '10000000000' + f_max: + - '1.0' + f_min: + - '1.0' + f_start: + - 1e-06 + warm_up_steps: + - '0' + enable_gradient_in_rollout: 'True' + enable_preprocessors: 'True' + fake_score: null + fake_score_net: + _target_: + audio_hidden_size: '32' + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors + in_dim: '65' + merge_lora: 'False' + mode: v2v + model_size: 1.3B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_phase2/step-19500.pt + schedule_type: rf + use_audio: 'True' + fake_score_optimizer: + _target_: + betas: + - '0.0' + - '0.999' + eps: 1e-08 + fused: 'False' + lr: 2e-06 + model: null + optim_type: adamw + weight_decay: '0.01' + fake_score_pred_type: x0 + fake_score_scheduler: + _target_: + cycle_lengths: + - '10000000000' + f_max: + - '1.0' + f_min: + - '1.0' + f_start: + - 1e-06 + warm_up_steps: + - '0' + fsdp_meta_init: 'False' + gan_loss_weight_gen: '0' + gan_r1_reg_alpha: '0.1' + gan_r1_reg_weight: '0.0' + gan_use_same_t_noise: 'False' + grad_scaler_enabled: 'False' + grad_scaler_growth_interval: '2000' + grad_scaler_init_scale: '65536.0' + guidance_scale: '4.5' + input_shape: + - '16' + - '21' + - '64' + - '64' + last_step_only: 'False' + load_student_weights: 'False' + net: + _target_: + audio_hidden_size: '32' + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors + chunk_size: '3' + in_dim: '65' + local_attn_size: '7' + mode: v2v + model_size: 1.3B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_1.3B_phase2/step-19500.pt + schedule_type: rf + sink_size: '1' + total_num_frames: '21' + use_audio: 'True' + use_dynamic_rope: 'True' + net_optimizer: + _target_: + betas: + - '0.0' + - '0.999' + eps: 1e-08 + fused: 'False' + lr: 2e-06 + model: null + optim_type: adamw + weight_decay: '0.01' + net_scheduler: + _target_: + cycle_lengths: + - '10000000000' + f_max: + - '1.0' + f_min: + - '1.0' + f_start: + - 1e-06 + warm_up_steps: + - '0' + precision: bfloat16 + precision_amp: null + precision_amp_enc: null + precision_amp_infer: null + precision_fsdp: float32 + pretrained_model_path: '' + pretrained_student_net_path: '' + reward: + audio_sample_rate: '16000' + checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model + decoder_kind: taew + enabled: 'True' + input_fps: '25.0' + taew_checkpoint_path: /home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth + vshift: '15' + reward_beta: '2' + reward_debug_dir: logs/redmd_debug + reward_weighting_mode: per_sample + same_step_across_blocks: 'True' + sample_t_cfg: + log_t_df: '0.01' + max_t: '0.999' + min_t: '0.001' + shift: '5.0' + t_list: + - '0.999' + - '0.937' + - '0.833' + - '0.624' + - '0.0' + time_dist_type: shifted + train_p_mean: '-1.1' + train_p_std: '2.0' + save_reward_debug_video: 'False' + skip_layers: null + start_gradient_frame: '0' + student_sample_steps: '4' + student_sample_type: sde + student_update_freq: '5' + teacher: + _target_: + audio_hidden_size: '32' + base_model_paths: /home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors + in_dim: '65' + merge_lora: 'True' + mode: v2v + model_size: 14B + net_pred_type: flow + omniavatar_ckpt_path: /home/work/output_omniavatar_v2v_phase2/step-10500.pt + schedule_type: rf + use_audio: 'True' + timestep_cfg: + enabled: 'True' + t_hi: '0.882' + t_lo: '0.556' + use_ema: 'False' +model_class: + _target_: + config: null +trainer: + augment_pipe: null + batch_size_global: null + callbacks: + ema: + _target_: + beta: '0.9999' + ema_halflife_kimg: '500' + ema_rampup_ratio: '0.05' + gamma: '16.97' + start_iter: '0' + type: constant + gpu_stats: + _target_: + every_n: '100' + grad_clip: + _target_: + grad_norm: '10.0' + model_key: net + param_count: + _target_: + train_profiler: + _target_: + every_n: '100' + wandb: + _target_: + fps: '25' + sample_logging_iter: '100' + checkpointer: + pretrained_ckpt_key_map: + net: net + pretrained_ckpt_path: /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_shift_5_4gpu_bs16_lr1e5_10000iter/checkpoints/0010000.pth + s3_container: s3://checkpoints/fastgen + s3_credential: ./credentials/s3.json + save_dir: /tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints + use_s3: 'False' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: 'False' + fsdp: 'True' + fsdp_cpu_offload: 'False' + fsdp_min_num_params: '10000000' + fsdp_sharding_group_size: null + global_vars: null + global_vars_val: + - null + grad_accum_rounds: '2' + logging_iter: '1' + max_iter: '5000' + offload_module_in_decoding: 'False' + resume: 'False' + save_ckpt_iter: '100' + seed: '0' + skip_initial_validation: 'True' + tf32_enabled: 'True' + val_seed: null + validation_iter: '100' + visualize_teacher: 'False' diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/debug-internal.log b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..687e621fbafdd3d33cb9721ca2e68bd40f535263 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/debug-internal.log @@ -0,0 +1,6 @@ +{"time":"2026-04-20T00:28:42.398011276+09:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-04-20T00:28:42.787548846+09:00","level":"INFO","msg":"stream: created new stream","id":"fxpzd9v7"} +{"time":"2026-04-20T00:28:42.787597435+09:00","level":"INFO","msg":"handler: started","stream_id":"fxpzd9v7"} +{"time":"2026-04-20T00:28:42.787663397+09:00","level":"INFO","msg":"stream: started","id":"fxpzd9v7"} +{"time":"2026-04-20T00:28:42.78768401+09:00","level":"INFO","msg":"sender: started","stream_id":"fxpzd9v7"} +{"time":"2026-04-20T00:28:42.787687913+09:00","level":"INFO","msg":"writer: started","stream_id":"fxpzd9v7"} diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/debug.log b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..b840db1d95308475597107b86844acbf528b90d9 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/debug.log @@ -0,0 +1,19 @@ +2026-04-20 00:28:42,139 INFO MainThread:2230976 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-04-20 00:28:42,139 INFO MainThread:2230976 [wandb_setup.py:_flush():81] Configure stats pid to 2230976 +2026-04-20 00:28:42,139 INFO MainThread:2230976 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-04-20 00:28:42,140 INFO MainThread:2230976 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/logs/debug.log +2026-04-20 00:28:42,140 INFO MainThread:2230976 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/logs/debug-internal.log +2026-04-20 00:28:42,140 INFO MainThread:2230976 [wandb_init.py:init():844] calling init triggers +2026-04-20 00:28:42,140 INFO MainThread:2230976 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'log_config': {'project': 'OmniAvatar-FastGen', 'group': 'omniavatar_sf_audiofix', 'name': 'sf_sink1_window7_redmd_audiofix_beta2_taew_4step', 'wandb_mode': 'online', 'wandb_entity': 'paulhcho', 'wandb_credential': './credentials/wandb_api.txt'}, 'trainer': {'cudnn': {'deterministic': 'False', 'benchmark': 'True'}, 'checkpointer': {'save_dir': '/tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints', 'use_s3': 'False', 's3_container': 's3://checkpoints/fastgen', 's3_credential': './credentials/s3.json', 'pretrained_ckpt_path': '/home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_shift_5_4gpu_bs16_lr1e5_10000iter/checkpoints/0010000.pth', 'pretrained_ckpt_key_map': {'net': 'net'}}, 'callbacks': {'grad_clip': {'grad_norm': '10.0', 'model_key': 'net', '_target_': ""}, 'gpu_stats': {'every_n': '100', '_target_': ""}, 'train_profiler': {'every_n': '100', '_target_': ""}, 'param_count': {'_target_': ""}, 'ema': {'type': 'constant', 'beta': '0.9999', 'gamma': '16.97', 'ema_halflife_kimg': '500', 'ema_rampup_ratio': '0.05', 'start_iter': '0', '_target_': ""}, 'wandb': {'sample_logging_iter': '100', '_target_': "", 'fps': '25'}}, 'save_ckpt_iter': '100', 'validation_iter': '100', 'skip_initial_validation': 'True', 'logging_iter': '1', 'max_iter': '5000', 'visualize_teacher': 'False', 'seed': '0', 'val_seed': None, 'resume': 'False', 'ddp': 'False', 'fsdp': 'True', 'tf32_enabled': 'True', 'grad_accum_rounds': '2', 'batch_size_global': None, 'offload_module_in_decoding': 'False', 'fsdp_cpu_offload': 'False', 'fsdp_min_num_params': '10000000', 'fsdp_sharding_group_size': None, 'global_vars': None, 'global_vars_val': [None], 'augment_pipe': None}, 'dataloader_train': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_path.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '8', 'num_workers': '2', 'neg_text_emb_path': None, 'use_ref_sequence': 'True', '_target_': "", 'load_raw_audio': 'True', 'raw_audio_sample_rate': '16000', 'raw_audio_num_frames': '81', 'raw_audio_fps': '25.0'}, 'dataloader_val': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_val10.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '1', 'num_workers': '2', 'neg_text_emb_path': None, 'use_ref_sequence': 'True', 'load_ode_path': 'False', '_target_': ''}, 'eval': {'num_samples': '50000', 'save_images': 'False', 'min_ckpt': '0', 'max_ckpt': '100000000', 'samples_dir': 'samples'}, 'model': {'net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'chunk_size': '3', 'total_num_frames': '21', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_phase2/step-19500.pt', 'net_pred_type': 'flow', 'schedule_type': 'rf', 'local_attn_size': '7', 'sink_size': '1', 'use_dynamic_rope': 'True', '_target_': ""}, 'teacher': {'model_size': '14B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_phase2/step-10500.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'fake_score_net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_phase2/step-19500.pt', 'merge_lora': 'False', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'guidance_scale': '4.5', 'timestep_cfg': {'enabled': 'True', 't_lo': '0.556', 't_hi': '0.882'}, 'skip_layers': None, 'net_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'net_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'sample_t_cfg': {'time_dist_type': 'shifted', 'train_p_mean': '-1.1', 'train_p_std': '2.0', 'shift': '5.0', 'min_t': '0.001', 'max_t': '0.999', 't_list': ['0.999', '0.937', '0.833', '0.624', '0.0'], 'log_t_df': '0.01'}, 'input_shape': ['16', '21', '64', '64'], 'device': 'cuda', 'grad_scaler_enabled': 'False', 'grad_scaler_init_scale': '65536.0', 'grad_scaler_growth_interval': '2000', 'pretrained_model_path': '', 'pretrained_student_net_path': '', 'load_student_weights': 'False', 'enable_preprocessors': 'True', 'use_ema': 'False', 'student_sample_steps': '4', 'student_sample_type': 'sde', 'fsdp_meta_init': 'False', 'add_teacher_to_fsdp_dict': 'True', 'ddp_find_unused_parameters': 'True', 'precision': 'bfloat16', 'precision_amp': None, 'precision_amp_infer': None, 'precision_amp_enc': None, 'precision_fsdp': 'float32', 'fake_score_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'fake_score_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'discriminator': {'feature_indices': '{0, 1, 2}', 'all_res': ['32', '16', '8'], 'in_channels': '256', '_target_': ""}, 'discriminator_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '0.0001', 'weight_decay': '0.01', 'betas': ['0.9', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'discriminator_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'student_update_freq': '5', 'gan_loss_weight_gen': '0', 'gan_use_same_t_noise': 'False', 'fake_score_pred_type': 'x0', 'gan_r1_reg_weight': '0.0', 'gan_r1_reg_alpha': '0.1', 'enable_gradient_in_rollout': 'True', 'start_gradient_frame': '0', 'same_step_across_blocks': 'True', 'last_step_only': 'False', 'context_noise': '0.0', 'fake_score': None, 'reward': {'enabled': 'True', 'checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model', 'input_fps': '25.0', 'audio_sample_rate': '16000', 'vshift': '15', 'decoder_kind': 'taew', 'taew_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth'}, 'reward_beta': '2', 'center_reward': 'False', 'clamp_reward': None, 'reward_weighting_mode': 'per_sample', 'save_reward_debug_video': 'False', 'reward_debug_dir': 'logs/redmd_debug'}, 'model_class': {'config': None, '_target_': ""}, '_wandb': {}} +2026-04-20 00:28:42,140 INFO MainThread:2230976 [wandb_init.py:init():892] starting backend +2026-04-20 00:28:42,383 INFO MainThread:2230976 [wandb_init.py:init():895] sending inform_init request +2026-04-20 00:28:42,396 INFO MainThread:2230976 [wandb_init.py:init():903] backend started and connected +2026-04-20 00:28:42,399 INFO MainThread:2230976 [wandb_init.py:init():973] updated telemetry +2026-04-20 00:28:42,421 INFO MainThread:2230976 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-04-20 00:28:43,491 INFO MainThread:2230976 [wandb_init.py:init():1042] starting run threads in backend +2026-04-20 00:28:43,785 INFO MainThread:2230976 [wandb_run.py:_console_start():2524] atexit reg +2026-04-20 00:28:43,785 INFO MainThread:2230976 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-04-20 00:28:43,785 INFO MainThread:2230976 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-04-20 00:28:43,785 INFO MainThread:2230976 [wandb_run.py:_redirect():2465] Redirects installed. +2026-04-20 00:28:43,790 INFO MainThread:2230976 [wandb_init.py:init():1082] run started, returning control to user process diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/train_media/data/real_100_47403c800591fce4a99b.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/train_media/data/real_100_47403c800591fce4a99b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23f2f377ccf704bcaa09a4c9b1a9b5151ab73aac --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/train_media/data/real_100_47403c800591fce4a99b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47403c800591fce4a99b38622b7607baeec11f4e025df3962d292acf880fba6c +size 192248 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/train_media/data/real_200_4e05a85e239f00a00e38.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/train_media/data/real_200_4e05a85e239f00a00e38.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c0fdfbad50d1423fb0ee98a75767d17d3c13b75b --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/train_media/data/real_200_4e05a85e239f00a00e38.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e05a85e239f00a00e386c8d8a9eaabeac26dd4943226f41d80d9f6b7c50bff6 +size 181830 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/train_media/data/real_300_97779d11e00d4fea84a0.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/train_media/data/real_300_97779d11e00d4fea84a0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1fd7a958d162e551e0f6ce9b94e075ce8f98807e --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/train_media/data/real_300_97779d11e00d4fea84a0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97779d11e00d4fea84a00b1667ca4d081ba87b85cf65fac40d228e9774f762d2 +size 194172 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/train_media/data/real_400_112c0905448a821974f0.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/train_media/data/real_400_112c0905448a821974f0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9684c7bde9ed18086d39e7a9880b642c505b3947 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/train_media/data/real_400_112c0905448a821974f0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:112c0905448a821974f0bf0a94d9107473bb02b60e0ebe1e75a57d309670facb +size 219311 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/train_media/data/real_500_2594e32c120a620ede9d.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/train_media/data/real_500_2594e32c120a620ede9d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7e38036c10e966c2c82fa717b29431a1ff88e1ae --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/train_media/data/real_500_2594e32c120a620ede9d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2594e32c120a620ede9d4613c87c8985d5ec03a9250077d94093cf8001922aae +size 190461 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/train_media/data/real_600_aebf0f3e2314a893fcf4.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/train_media/data/real_600_aebf0f3e2314a893fcf4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..df584863079059c25acb730ed404066fdef5d340 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/train_media/data/real_600_aebf0f3e2314a893fcf4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aebf0f3e2314a893fcf47216a0d6c68b02b30d80a7347c759681d9497a805c01 +size 220015 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/train_media/student/generation_100_dbed78ee0fdec1d90f2e.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/train_media/student/generation_100_dbed78ee0fdec1d90f2e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e86b3e0b473c21728278bb719588120363469648 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/train_media/student/generation_100_dbed78ee0fdec1d90f2e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dbed78ee0fdec1d90f2e010bbaada006a2e539d5c4ef5ed6c33df63558cd467c +size 192482 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/train_media/student/generation_200_f1ca6fb9f7ed2cf17ff3.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/train_media/student/generation_200_f1ca6fb9f7ed2cf17ff3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2e3d5983f2984bc083aa2f4a682774ce143b73c0 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/train_media/student/generation_200_f1ca6fb9f7ed2cf17ff3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f1ca6fb9f7ed2cf17ff30286b30bc97d15403ca9351f02429150a9f8cb2d5258 +size 199020 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/train_media/student/generation_300_ccd008653d34c4e1079d.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/train_media/student/generation_300_ccd008653d34c4e1079d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f7a30802d68712f3a380eed9581090974f42e63d --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/train_media/student/generation_300_ccd008653d34c4e1079d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ccd008653d34c4e1079da17d79450658f91d94476d0a9fd40853d45781085324 +size 221627 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/train_media/student/generation_400_607d6362f8aa8d1a9233.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/train_media/student/generation_400_607d6362f8aa8d1a9233.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f08c8cf30ff9655f94c192b08cb0fe4f73031206 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/train_media/student/generation_400_607d6362f8aa8d1a9233.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:607d6362f8aa8d1a9233f303d97dcbd41aaf334b3fa9c27d562c2beeba71df4a +size 245570 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/train_media/student/generation_500_1338a835b20737523158.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/train_media/student/generation_500_1338a835b20737523158.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..de3cbcd63d0e1dace40f129674fdea906c079b8c --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/train_media/student/generation_500_1338a835b20737523158.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1338a835b20737523158ab3eeaaffc530787eee69e071ddc15ee168742b4c580 +size 205447 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/train_media/student/generation_600_f8fe5f5817f741e0fa2e.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/train_media/student/generation_600_f8fe5f5817f741e0fa2e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c2de8727958bb56872e29a683daa6b9ce4b78118 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/train_media/student/generation_600_f8fe5f5817f741e0fa2e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8fe5f5817f741e0fa2e950f6a974ad64a3d02c4ba922824cf49b4c74c9492a6 +size 268532 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_100_065388816206e3899e69.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_100_065388816206e3899e69.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9bcfd8efcf66b073b338de3539e54253f00c9646 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_100_065388816206e3899e69.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:065388816206e3899e69af52887c2fb712f31aa64d63cd27ef3b3c6345f3742c +size 181391 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_100_22be8aa64c3275adc278.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_100_22be8aa64c3275adc278.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..66dd6d70a2f42b951d85fa440e5861bf1eca925c --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_100_22be8aa64c3275adc278.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22be8aa64c3275adc278ec2ede876a2fa57a15672dcf6701179cdc30af1abfdf +size 185236 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_100_629f5e819c61eb78324e.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_100_629f5e819c61eb78324e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4781732c3ae1e77d656851592da047ea952f099e --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_100_629f5e819c61eb78324e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:629f5e819c61eb78324ec669c2ff4e10fb91a31d0e3d35012c36f6e7a12c4c3a +size 226776 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_100_6906457b94f1950af020.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_100_6906457b94f1950af020.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7807a9a8276bcdcd3b23b9b4fb449c429409cf71 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_100_6906457b94f1950af020.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6906457b94f1950af020028234c2e5f6445fd6c3f21105cba7b73ba90064dbb8 +size 194704 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_100_80ab16039c803c1187c0.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_100_80ab16039c803c1187c0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..150046672f9d5e2d89383b6491ce8f39a96e81e0 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_100_80ab16039c803c1187c0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80ab16039c803c1187c0f2cfa50731120dafce4ee942c285bad10921a6ea803f +size 165744 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_100_83109bf9fb68148a0b72.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_100_83109bf9fb68148a0b72.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6896820250ca15008c5d14348b23a8bfb3cc5d52 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_100_83109bf9fb68148a0b72.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83109bf9fb68148a0b723df7273b8d58378decf9d3fb3551136a61297a06f2fd +size 248419 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_100_882bdb3096251d76a010.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_100_882bdb3096251d76a010.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0925d3a898ad0c6bfc7513e46fa58ae5c081ec1a --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_100_882bdb3096251d76a010.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:882bdb3096251d76a010cd69b2a006797bd2a2ff3e03a9b801283fb3deec0e04 +size 177571 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_100_9593187e51a6d7571ed2.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_100_9593187e51a6d7571ed2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8b5ad9864a971f7fddce86e1de7c48ac847a2836 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_100_9593187e51a6d7571ed2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9593187e51a6d7571ed200c6c412e8f5c4bff5f2b2ea1f9e4149492062e66d06 +size 226738 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_100_a35a541a84c35b1c8ca4.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_100_a35a541a84c35b1c8ca4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c2e06eed8aea58f197e0c969a975f3add21950d9 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_100_a35a541a84c35b1c8ca4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a35a541a84c35b1c8ca45b5e95818db3058299c1b402747b4b4750c2462b8b0c +size 225213 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_100_d438e3becfef4d908558.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_100_d438e3becfef4d908558.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..abfea7b7f20627e4c991ec2400e515547fae8544 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_100_d438e3becfef4d908558.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d438e3becfef4d908558f6f226f33b8022cfae2b2c2a5f6cbcdf8e07e31fb788 +size 275406 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_200_08b05745233fe6193330.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_200_08b05745233fe6193330.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..77376475dd7c1befbc530a3dbff61497424f5815 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_200_08b05745233fe6193330.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08b05745233fe6193330f7913438757ee733414c22c8bd676e5ec74f078a2688 +size 199806 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_200_473440a7b653e2454a25.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_200_473440a7b653e2454a25.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7e75b77e444f64579ba7ff30074f534335eef710 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_200_473440a7b653e2454a25.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:473440a7b653e2454a256e96458059cb7996438672ae09d8b510b054257227e7 +size 171685 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_200_6fdcaae10a79aa2872ec.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_200_6fdcaae10a79aa2872ec.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f4a24861340e2889bbcf1c89d30630c3a42e54fc --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_200_6fdcaae10a79aa2872ec.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6fdcaae10a79aa2872ec2d681310c750ae05c941d972f43cfce320205ec60ab2 +size 235157 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_200_7dcf75e9215bb38e7093.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_200_7dcf75e9215bb38e7093.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..458ad77ddd706a0eec944d683648b6665e855fc3 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_200_7dcf75e9215bb38e7093.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7dcf75e9215bb38e70930e9df7bd8bc14d67cc9ddb4921e4da08d3387f819fbc +size 285125 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_200_7f4db1587fc269fd3729.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_200_7f4db1587fc269fd3729.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3105d844e100764eae48d6278744eb7a03f64b0a --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_200_7f4db1587fc269fd3729.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f4db1587fc269fd372928e178453dbaacc1b52dd79e194b1ecb499f4b1cb2f6 +size 185166 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_200_a455d874652f89da8df7.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_200_a455d874652f89da8df7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ddedd4dde1752b09d232aae138013ee6f6a1bec7 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_200_a455d874652f89da8df7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a455d874652f89da8df7bc13aa4eeea65cd4824a1051831da2f23bd45302706d +size 261255 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_200_abfcbdfc4ceabd160d2f.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_200_abfcbdfc4ceabd160d2f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c7e270b383f48b36e7128d28b38ab87fb9e680b2 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_200_abfcbdfc4ceabd160d2f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:abfcbdfc4ceabd160d2f18b7ae89c5c3bd99f6f5498648d5ac6041d2af583413 +size 193192 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_200_b0cf3c464e74a54def67.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_200_b0cf3c464e74a54def67.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fffef0ae931323237db23d4fb64a61f9c83e0438 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_200_b0cf3c464e74a54def67.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0cf3c464e74a54def673cd994ed573a50322e1b25b11fd10594df1bcbac563a +size 234089 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_200_b836dd841434b50dbb1e.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_200_b836dd841434b50dbb1e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..31eea04bd286f056d3c6608317ad63a184a45227 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_200_b836dd841434b50dbb1e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b836dd841434b50dbb1eb0b0d7c7ebbe061f88756fdc6d086824ce75cc1c4bd1 +size 187804 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_200_d280cac5fa1c9a11802d.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_200_d280cac5fa1c9a11802d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7d5a2d5f11c646fcbf28da56454e715a6f310804 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_200_d280cac5fa1c9a11802d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d280cac5fa1c9a11802d36d0a3e73dfeb87fa6065b0b6db8075e96cd4ff01a35 +size 237348 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_300_0df52ffc1f51a11a274f.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_300_0df52ffc1f51a11a274f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6c92fa37d76c3ff39a505faa52748a5a001a617f --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_300_0df52ffc1f51a11a274f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0df52ffc1f51a11a274fa195d0be3ae343c476e6a144339ce92431ca6ef7e4c8 +size 182927 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_300_1fb9504f0da3470e5499.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_300_1fb9504f0da3470e5499.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ee1e8809afe668149fe054b45b5fe434fe43c7a0 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_300_1fb9504f0da3470e5499.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1fb9504f0da3470e5499f8e6655c4da9d74767554a95bd3242183db9ddf0c46a +size 249745 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_300_5ad6839552a5c758173d.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_300_5ad6839552a5c758173d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..02a5c022e662395ca6c4a7f243534e56b8fcf4e1 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_300_5ad6839552a5c758173d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ad6839552a5c758173d431d7bb9feb4c520ff7df549c2f592ab10cee2bfb8a3 +size 255100 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_300_6d315d601d30bcf4f5ff.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_300_6d315d601d30bcf4f5ff.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..385a0f19a80e504c7a7c670b6ed017439698e146 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_300_6d315d601d30bcf4f5ff.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d315d601d30bcf4f5ff4998d393ac06fb5ff9a8efa74a30155f4a8beec18216 +size 213366 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_300_9ff25f0e613106da8677.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_300_9ff25f0e613106da8677.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..92cfa72b50571307defa90486b515aaae409ffc9 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_300_9ff25f0e613106da8677.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ff25f0e613106da867713e2bf9684de642fca0be7530822f52fb02b402c4543 +size 207414 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_300_a6ff81de78f80198f4e8.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_300_a6ff81de78f80198f4e8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..61693143e84751feb8334e1cc36866e63800c609 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_300_a6ff81de78f80198f4e8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a6ff81de78f80198f4e8d6328dded874c95b85ded24d2f2742cb950ea4f89241 +size 251015 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_300_bb5e13235464274ec0a4.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_300_bb5e13235464274ec0a4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..21abfe7cd6ce965a030be942fc818e6d8ca7ea86 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_300_bb5e13235464274ec0a4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb5e13235464274ec0a46d0a01fed2ddb2a4443b27118af054d095929172d804 +size 277974 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_300_cef4b9d308aebb76139c.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_300_cef4b9d308aebb76139c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1e2a934b7b9fbd603ad8d04ac9fe219968669b3c --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_300_cef4b9d308aebb76139c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cef4b9d308aebb76139c0c7a63e0bbc46366db608cd0a51064220e7a193ef7bb +size 293892 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_300_d4dabdf6ff31d17a324a.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_300_d4dabdf6ff31d17a324a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..16d39443641c7ea555f8311350104b060d96a418 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_300_d4dabdf6ff31d17a324a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4dabdf6ff31d17a324a9683a21ab4ab8379493509fa0d8c4866fe24e1b2e072 +size 201519 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_300_d86650f7bc1e730c3ee1.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_300_d86650f7bc1e730c3ee1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e0c44e537c5d3ad359996a113aa81f70a0de9cc2 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_300_d86650f7bc1e730c3ee1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d86650f7bc1e730c3ee105405bc226ff8cc865441f9331a170d5146a5cbc3081 +size 196381 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_400_079a3388458485b0ce5f.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_400_079a3388458485b0ce5f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..663f4817dd05ff03a8e983a743998e4ceb5268ec --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_400_079a3388458485b0ce5f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:079a3388458485b0ce5f8da40f34ed5b8de797a7e12b2aa6417e1faaf9617ec5 +size 237898 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_400_272ad7a28e6015af5078.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_400_272ad7a28e6015af5078.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c7b58884b2ccc0a8e98f47feea97107ef27c24c8 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_400_272ad7a28e6015af5078.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:272ad7a28e6015af507826420c69b9d3a09070e9cf796b74871b1b0b4d887e7d +size 241322 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_400_470a673634ca0778a523.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_400_470a673634ca0778a523.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9884c07142042443dee0054625eec2c4dd4872f2 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_400_470a673634ca0778a523.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:470a673634ca0778a523f3ccad59fe3ef6641e4554c6827cc72b33e51adcd14e +size 179150 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_400_5d7d48636aef730f2fef.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_400_5d7d48636aef730f2fef.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e49a0f9085501904c0c9a0634ea46c0651dbbbc7 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_400_5d7d48636aef730f2fef.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d7d48636aef730f2fefc11b7976847500df363906c8cb2236cbdf3ff2e6ee2c +size 200394 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_400_671e25a6f515483157b7.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_400_671e25a6f515483157b7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7fd0b35da83b44ecc8c37f59cb5b17c544a6a3a3 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_400_671e25a6f515483157b7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:671e25a6f515483157b7ea9a5b7c0fcc44f358cbf179027ac9275b5e94729dda +size 272736 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_400_7bea112416b01940b4f4.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_400_7bea112416b01940b4f4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ba1d0f8ebdcce663db1d8ea73904144c0bbbf6bb --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_400_7bea112416b01940b4f4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7bea112416b01940b4f43d68adb750eb92a7a4a2d1d88cbdf214067ba9824b0a +size 194802 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_400_7f84241ae237b80cf500.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_400_7f84241ae237b80cf500.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c11565887a3079ff8ffbcf9283aa55b790e08384 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_400_7f84241ae237b80cf500.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f84241ae237b80cf5007dea51218f24f385944cff6e62a98de7dd21d589f168 +size 189361 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_400_d0dc14a28c02520d2844.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_400_d0dc14a28c02520d2844.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..29cace96eb16574cd5d04d0e953d9536fb9aa867 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_400_d0dc14a28c02520d2844.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0dc14a28c02520d2844978b8091150ce0bddb554074a186a1508da8b59bc4a0 +size 287225 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_400_d23a2ae68b13caed8e64.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_400_d23a2ae68b13caed8e64.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..acba8f8734dd45c9d0f3f88f5fd1b355672b5eed --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_400_d23a2ae68b13caed8e64.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d23a2ae68b13caed8e649bb7d105444cb0ba81877a3db1fd2b740ede88564504 +size 206524 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_400_f3ed3833ea95e0d907b5.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_400_f3ed3833ea95e0d907b5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..584b1013d764f92ba21015dfb3ae4d75643d0a97 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_400_f3ed3833ea95e0d907b5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3ed3833ea95e0d907b50153f4fc6b52f5ea2a6c8ab44eb01f2d01e4189f0b9f +size 242978 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_500_0d712ca5c3286b44ef10.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_500_0d712ca5c3286b44ef10.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a974345efe77b5b8dc0c75bcbf6337123c67c5a7 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_500_0d712ca5c3286b44ef10.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d712ca5c3286b44ef1034a5a6780ce302f5bb29e81ea5487c82ecd4c1816b25 +size 189278 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_500_157dfe635f0203c31db0.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_500_157dfe635f0203c31db0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2a130865738e4932e98d229267c661bf9c34b877 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_500_157dfe635f0203c31db0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:157dfe635f0203c31db0e15f1bbe71a22dbaeba583a0004b4aff5a433524e366 +size 209505 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_500_47537149d882f627a6c9.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_500_47537149d882f627a6c9.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a6cf253a75bed498b53737126a9f248f8f2dbd1e --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_500_47537149d882f627a6c9.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47537149d882f627a6c94e18416dbea1b34bb785fbaaa60bdbac1b088ae79d21 +size 236608 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_500_561404fbd13b0dc9f796.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_500_561404fbd13b0dc9f796.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9e819a67cf1a442241ffe9452c7f0825da0c4148 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_500_561404fbd13b0dc9f796.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:561404fbd13b0dc9f7965fddfa8652a89fa3c56eafc6c0d4f5b4acb43b38e594 +size 200617 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_500_7e7a3a035a1c7faf3b7c.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_500_7e7a3a035a1c7faf3b7c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..15364d34f72a25290ef304c02bcfcea1dfa68194 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_500_7e7a3a035a1c7faf3b7c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e7a3a035a1c7faf3b7cc87c0cd6ffa32db3285aca45b83683ee68229c7726a7 +size 269848 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_500_7f1d9130a8ce38f683f1.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_500_7f1d9130a8ce38f683f1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..315d2b5621db05b5c84d79890709c2479d7b40be --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_500_7f1d9130a8ce38f683f1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f1d9130a8ce38f683f1f0381ae6da3f49cf15e56868e340dedd701b7cf625b8 +size 176817 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_500_80b5402cd70bbb987599.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_500_80b5402cd70bbb987599.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f4e45876c4dac57f7b7f8289958893e02733b486 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_500_80b5402cd70bbb987599.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80b5402cd70bbb987599501292dd56ac604f2c4ecb1e7314423c75e9478d8822 +size 285761 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_500_99ba44fb1c4871734865.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_500_99ba44fb1c4871734865.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..970b1b00eae63f1e24b5405ee871a39f4bc33c54 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_500_99ba44fb1c4871734865.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99ba44fb1c487173486500cd743058bac1593276e8afccf330270d541ac3075c +size 241665 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_500_de98e19993c9db197ada.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_500_de98e19993c9db197ada.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6a5dc31198467bda7f4d9045ed457e0c44c6c7a7 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_500_de98e19993c9db197ada.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de98e19993c9db197ada712da70c16f06b3166ce227c010d00e610563913ba0a +size 193565 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_500_ec6f3115c2698671e276.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_500_ec6f3115c2698671e276.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bbe956b7d5b0bc14c9c8a887e7445742336ef039 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_500_ec6f3115c2698671e276.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec6f3115c2698671e2769d79ec1b91dd1f6062be064569f2c1cb7e60b1b54a7c +size 241762 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_600_117f332517d46edf01fd.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_600_117f332517d46edf01fd.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ede240887a2cac80fe3d41421a98106e0c6d8aab --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_600_117f332517d46edf01fd.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:117f332517d46edf01fde625e361833aed579403f97674a4580d9650c49507f4 +size 203998 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_600_1db9359d35ea2edc77ef.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_600_1db9359d35ea2edc77ef.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8c9925d02d3206d479f6ade772d38139106d9dc6 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_600_1db9359d35ea2edc77ef.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1db9359d35ea2edc77ef3482aa76347b57ad260bc5546bb70a338b19110fa409 +size 206525 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_600_3b016fb2a032e42c4746.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_600_3b016fb2a032e42c4746.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..283ea44e7b5b4dc6a98664442715d12e59a07c6f --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_600_3b016fb2a032e42c4746.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b016fb2a032e42c4746818811a1151438ac03de98b4f6dc983b9c3a67e6a5c6 +size 227623 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_600_7027ebe09b83c2128696.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_600_7027ebe09b83c2128696.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ec240e95be310399c87812bf2e01d6238a59fe05 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_600_7027ebe09b83c2128696.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7027ebe09b83c2128696824f86e3360470ffe79ad9841abe4030b622e2271b85 +size 303627 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_600_714d74f6d5a878a7b6db.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_600_714d74f6d5a878a7b6db.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0bc116a581a5a8834d2be8372f09d65f60d6a5e5 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_600_714d74f6d5a878a7b6db.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:714d74f6d5a878a7b6db033406bbcc015b074226f58cbd4dae5829ac5c3391ae +size 289164 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_600_76db7dc840b20b5fc96c.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_600_76db7dc840b20b5fc96c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5da1b418c244f83be234531e93963cdd853e660f --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_600_76db7dc840b20b5fc96c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76db7dc840b20b5fc96c01fa684596dd478a6d0b402109a6e83d85e1c04cbc42 +size 184219 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_600_877c5e326d3373357f9f.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_600_877c5e326d3373357f9f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ebf6598c5943521c35c5944d88cdfd6695cf5c82 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_600_877c5e326d3373357f9f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:877c5e326d3373357f9ff23964ee23f7c1b9b778ed6d234e67a3a2dbe3a3d09d +size 215098 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_600_be7cb5d2d76a5b1db3ef.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_600_be7cb5d2d76a5b1db3ef.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ab501cfff117eef433f2ecaf9cf8053f63eb1932 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_600_be7cb5d2d76a5b1db3ef.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be7cb5d2d76a5b1db3ef4f9ae67689bb67bb83fbd4fca30f211d845132952f97 +size 254610 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_600_fcf497d71dd93a3aefa9.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_600_fcf497d71dd93a3aefa9.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5b0d1f563d36e78bc3c18b29020a345dc123c0c1 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_600_fcf497d71dd93a3aefa9.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fcf497d71dd93a3aefa9a63f0b4111e1dde46fee93aac2e8a42e54229b30c94b +size 252474 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_600_fd09907d49b2ad927921.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_600_fd09907d49b2ad927921.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..626ae5f65d77ca1c457972d68bbfda9787983ed8 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/generated_600_fd09907d49b2ad927921.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd09907d49b2ad92792190dac322ff1a261a5af4245d9863c3817acc7b006d35 +size 258480 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_100_15ac5b765a401d035db1.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_100_15ac5b765a401d035db1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39d693cdae1da9a3a67729fa63439503876eafbc --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_100_15ac5b765a401d035db1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117 +size 190004 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_100_294040ac378ceaa34329.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_100_294040ac378ceaa34329.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23dc14931dfca40fe3f7d6b1fdefe0a1edca1ab9 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_100_294040ac378ceaa34329.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c +size 249372 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_100_6ff046418b5d41477989.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_100_6ff046418b5d41477989.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95ab423dcc64a40477b5d8f2e5d44351e2781109 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_100_6ff046418b5d41477989.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac +size 214001 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_100_80aab00fcfa85d5db98e.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_100_80aab00fcfa85d5db98e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09ef3a36f7d3e07e8b458916212891367b079325 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_100_80aab00fcfa85d5db98e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327 +size 182028 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_100_884b2d197a5511393680.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_100_884b2d197a5511393680.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..550cb7c37290fce2f4554ad8f26113320700bfa3 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_100_884b2d197a5511393680.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214 +size 170922 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_100_a076d17e202ed18ebdf8.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_100_a076d17e202ed18ebdf8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c59919de964bcf66a3cfe53f52bedd0a8b91df2e --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_100_a076d17e202ed18ebdf8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb +size 165516 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_100_c0196720a45889bb2198.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_100_c0196720a45889bb2198.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018bd36397d571746795d1f4619cc8f4fc2766da --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_100_c0196720a45889bb2198.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8 +size 173236 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_100_c0f892b5fba78348af85.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_100_c0f892b5fba78348af85.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f296c5f3aec5b609e307e17bbeffa25264deb91 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_100_c0f892b5fba78348af85.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8 +size 222986 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_100_f5463a4ea6f68a39c5d0.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_100_f5463a4ea6f68a39c5d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..078934eedc27e8de1e5fed0272335f707868e553 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_100_f5463a4ea6f68a39c5d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395 +size 164630 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_100_f7412c9e170ffd299748.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_100_f7412c9e170ffd299748.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eaf7bba0c634cee305854ae2bbd756c90092d38 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_100_f7412c9e170ffd299748.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273 +size 227490 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_200_15ac5b765a401d035db1.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_200_15ac5b765a401d035db1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39d693cdae1da9a3a67729fa63439503876eafbc --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_200_15ac5b765a401d035db1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117 +size 190004 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_200_294040ac378ceaa34329.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_200_294040ac378ceaa34329.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23dc14931dfca40fe3f7d6b1fdefe0a1edca1ab9 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_200_294040ac378ceaa34329.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c +size 249372 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_200_6ff046418b5d41477989.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_200_6ff046418b5d41477989.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95ab423dcc64a40477b5d8f2e5d44351e2781109 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_200_6ff046418b5d41477989.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac +size 214001 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_200_80aab00fcfa85d5db98e.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_200_80aab00fcfa85d5db98e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09ef3a36f7d3e07e8b458916212891367b079325 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_200_80aab00fcfa85d5db98e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327 +size 182028 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_200_884b2d197a5511393680.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_200_884b2d197a5511393680.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..550cb7c37290fce2f4554ad8f26113320700bfa3 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_200_884b2d197a5511393680.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214 +size 170922 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_200_a076d17e202ed18ebdf8.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_200_a076d17e202ed18ebdf8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c59919de964bcf66a3cfe53f52bedd0a8b91df2e --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_200_a076d17e202ed18ebdf8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb +size 165516 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_200_c0196720a45889bb2198.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_200_c0196720a45889bb2198.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018bd36397d571746795d1f4619cc8f4fc2766da --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_200_c0196720a45889bb2198.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8 +size 173236 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_200_c0f892b5fba78348af85.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_200_c0f892b5fba78348af85.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f296c5f3aec5b609e307e17bbeffa25264deb91 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_200_c0f892b5fba78348af85.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8 +size 222986 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_200_f5463a4ea6f68a39c5d0.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_200_f5463a4ea6f68a39c5d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..078934eedc27e8de1e5fed0272335f707868e553 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_200_f5463a4ea6f68a39c5d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395 +size 164630 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_200_f7412c9e170ffd299748.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_200_f7412c9e170ffd299748.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eaf7bba0c634cee305854ae2bbd756c90092d38 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_200_f7412c9e170ffd299748.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273 +size 227490 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_300_15ac5b765a401d035db1.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_300_15ac5b765a401d035db1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39d693cdae1da9a3a67729fa63439503876eafbc --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_300_15ac5b765a401d035db1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117 +size 190004 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_300_294040ac378ceaa34329.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_300_294040ac378ceaa34329.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23dc14931dfca40fe3f7d6b1fdefe0a1edca1ab9 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_300_294040ac378ceaa34329.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c +size 249372 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_300_6ff046418b5d41477989.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_300_6ff046418b5d41477989.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95ab423dcc64a40477b5d8f2e5d44351e2781109 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_300_6ff046418b5d41477989.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac +size 214001 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_300_80aab00fcfa85d5db98e.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_300_80aab00fcfa85d5db98e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09ef3a36f7d3e07e8b458916212891367b079325 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_300_80aab00fcfa85d5db98e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327 +size 182028 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_300_884b2d197a5511393680.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_300_884b2d197a5511393680.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..550cb7c37290fce2f4554ad8f26113320700bfa3 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_300_884b2d197a5511393680.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214 +size 170922 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_300_a076d17e202ed18ebdf8.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_300_a076d17e202ed18ebdf8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c59919de964bcf66a3cfe53f52bedd0a8b91df2e --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_300_a076d17e202ed18ebdf8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb +size 165516 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_300_c0196720a45889bb2198.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_300_c0196720a45889bb2198.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018bd36397d571746795d1f4619cc8f4fc2766da --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_300_c0196720a45889bb2198.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8 +size 173236 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_300_c0f892b5fba78348af85.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_300_c0f892b5fba78348af85.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f296c5f3aec5b609e307e17bbeffa25264deb91 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_300_c0f892b5fba78348af85.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8 +size 222986 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_300_f5463a4ea6f68a39c5d0.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_300_f5463a4ea6f68a39c5d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..078934eedc27e8de1e5fed0272335f707868e553 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_300_f5463a4ea6f68a39c5d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395 +size 164630 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_300_f7412c9e170ffd299748.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_300_f7412c9e170ffd299748.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eaf7bba0c634cee305854ae2bbd756c90092d38 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_300_f7412c9e170ffd299748.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273 +size 227490 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_400_15ac5b765a401d035db1.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_400_15ac5b765a401d035db1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39d693cdae1da9a3a67729fa63439503876eafbc --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_400_15ac5b765a401d035db1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117 +size 190004 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_400_294040ac378ceaa34329.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_400_294040ac378ceaa34329.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23dc14931dfca40fe3f7d6b1fdefe0a1edca1ab9 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_400_294040ac378ceaa34329.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c +size 249372 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_400_6ff046418b5d41477989.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_400_6ff046418b5d41477989.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95ab423dcc64a40477b5d8f2e5d44351e2781109 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_400_6ff046418b5d41477989.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac +size 214001 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_400_80aab00fcfa85d5db98e.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_400_80aab00fcfa85d5db98e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09ef3a36f7d3e07e8b458916212891367b079325 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_400_80aab00fcfa85d5db98e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327 +size 182028 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_400_884b2d197a5511393680.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_400_884b2d197a5511393680.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..550cb7c37290fce2f4554ad8f26113320700bfa3 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_400_884b2d197a5511393680.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214 +size 170922 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_400_a076d17e202ed18ebdf8.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_400_a076d17e202ed18ebdf8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c59919de964bcf66a3cfe53f52bedd0a8b91df2e --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_400_a076d17e202ed18ebdf8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb +size 165516 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_400_c0196720a45889bb2198.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_400_c0196720a45889bb2198.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018bd36397d571746795d1f4619cc8f4fc2766da --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_400_c0196720a45889bb2198.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8 +size 173236 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_400_c0f892b5fba78348af85.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_400_c0f892b5fba78348af85.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f296c5f3aec5b609e307e17bbeffa25264deb91 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_400_c0f892b5fba78348af85.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8 +size 222986 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_400_f5463a4ea6f68a39c5d0.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_400_f5463a4ea6f68a39c5d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..078934eedc27e8de1e5fed0272335f707868e553 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_400_f5463a4ea6f68a39c5d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395 +size 164630 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_400_f7412c9e170ffd299748.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_400_f7412c9e170ffd299748.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eaf7bba0c634cee305854ae2bbd756c90092d38 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_400_f7412c9e170ffd299748.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273 +size 227490 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_500_15ac5b765a401d035db1.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_500_15ac5b765a401d035db1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39d693cdae1da9a3a67729fa63439503876eafbc --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_500_15ac5b765a401d035db1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117 +size 190004 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_500_294040ac378ceaa34329.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_500_294040ac378ceaa34329.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23dc14931dfca40fe3f7d6b1fdefe0a1edca1ab9 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_500_294040ac378ceaa34329.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c +size 249372 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_500_6ff046418b5d41477989.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_500_6ff046418b5d41477989.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95ab423dcc64a40477b5d8f2e5d44351e2781109 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_500_6ff046418b5d41477989.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac +size 214001 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_500_80aab00fcfa85d5db98e.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_500_80aab00fcfa85d5db98e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09ef3a36f7d3e07e8b458916212891367b079325 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_500_80aab00fcfa85d5db98e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327 +size 182028 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_500_884b2d197a5511393680.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_500_884b2d197a5511393680.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..550cb7c37290fce2f4554ad8f26113320700bfa3 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_500_884b2d197a5511393680.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214 +size 170922 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_500_a076d17e202ed18ebdf8.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_500_a076d17e202ed18ebdf8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c59919de964bcf66a3cfe53f52bedd0a8b91df2e --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_500_a076d17e202ed18ebdf8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb +size 165516 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_500_c0196720a45889bb2198.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_500_c0196720a45889bb2198.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018bd36397d571746795d1f4619cc8f4fc2766da --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_500_c0196720a45889bb2198.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8 +size 173236 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_500_c0f892b5fba78348af85.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_500_c0f892b5fba78348af85.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f296c5f3aec5b609e307e17bbeffa25264deb91 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_500_c0f892b5fba78348af85.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8 +size 222986 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_500_f5463a4ea6f68a39c5d0.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_500_f5463a4ea6f68a39c5d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..078934eedc27e8de1e5fed0272335f707868e553 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_500_f5463a4ea6f68a39c5d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395 +size 164630 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_500_f7412c9e170ffd299748.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_500_f7412c9e170ffd299748.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eaf7bba0c634cee305854ae2bbd756c90092d38 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_500_f7412c9e170ffd299748.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273 +size 227490 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_600_15ac5b765a401d035db1.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_600_15ac5b765a401d035db1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39d693cdae1da9a3a67729fa63439503876eafbc --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_600_15ac5b765a401d035db1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ac5b765a401d035db14582a0a32494ae6fcbd78ed5cfb619eb35c784e59117 +size 190004 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_600_294040ac378ceaa34329.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_600_294040ac378ceaa34329.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23dc14931dfca40fe3f7d6b1fdefe0a1edca1ab9 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_600_294040ac378ceaa34329.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294040ac378ceaa3432978b51a8207f6475cc2ebfc4949b87fd773068402e60c +size 249372 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_600_6ff046418b5d41477989.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_600_6ff046418b5d41477989.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..95ab423dcc64a40477b5d8f2e5d44351e2781109 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_600_6ff046418b5d41477989.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff046418b5d4147798961c8e246d029bf98d6d4023d8effe3f7ee226f6416ac +size 214001 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_600_80aab00fcfa85d5db98e.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_600_80aab00fcfa85d5db98e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09ef3a36f7d3e07e8b458916212891367b079325 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_600_80aab00fcfa85d5db98e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80aab00fcfa85d5db98e73f0e723fc022b7961a4da020b311fa2cfa010ced327 +size 182028 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_600_884b2d197a5511393680.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_600_884b2d197a5511393680.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..550cb7c37290fce2f4554ad8f26113320700bfa3 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_600_884b2d197a5511393680.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884b2d197a5511393680bd846510f6dbf533353993bcbed2bae2d03234611214 +size 170922 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_600_a076d17e202ed18ebdf8.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_600_a076d17e202ed18ebdf8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c59919de964bcf66a3cfe53f52bedd0a8b91df2e --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_600_a076d17e202ed18ebdf8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a076d17e202ed18ebdf897ca1623592290eb9cdbee34e0eadb9c46774dc31fdb +size 165516 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_600_c0196720a45889bb2198.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_600_c0196720a45889bb2198.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018bd36397d571746795d1f4619cc8f4fc2766da --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_600_c0196720a45889bb2198.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0196720a45889bb219825d0a1691eb71eae3c73fa84c3394947f4d1b1858ba8 +size 173236 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_600_c0f892b5fba78348af85.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_600_c0f892b5fba78348af85.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f296c5f3aec5b609e307e17bbeffa25264deb91 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_600_c0f892b5fba78348af85.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f892b5fba78348af85352209b3b58ab4e2a1f114343d592e09e3a0c3385dc8 +size 222986 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_600_f5463a4ea6f68a39c5d0.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_600_f5463a4ea6f68a39c5d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..078934eedc27e8de1e5fed0272335f707868e553 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_600_f5463a4ea6f68a39c5d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5463a4ea6f68a39c5d012df793fc6f0f240f8cb1bbcaba89b6588cd3223c395 +size 164630 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_600_f7412c9e170ffd299748.mp4 b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_600_f7412c9e170ffd299748.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eaf7bba0c634cee305854ae2bbd756c90092d38 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/media/videos/val0/reconstructed_600_f7412c9e170ffd299748.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7412c9e170ffd2997489a3a14d37d0260b7df6cab5b53a709594ad505782273 +size 227490 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/output.log b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..86f41ee8036761e0b0ebcaa51dd2f16139f7f5b6 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/output.log @@ -0,0 +1,12768 @@ +[Apr 20, 2026 - 00:28:43 | INFO | fastgen.callbacks.wandb:init_wandb:259 ] Wandb code upload disabled (set WANDB_UPLOAD_CODE=true to enable) +[Apr 20, 2026 - 00:28:43 | SUCCESS | fastgen.trainer:__init__:53 ] Callbacks initialized successfully +[Apr 20, 2026 - 00:28:43 | INFO | fastgen.trainer:__init__:57 ] Callback synchronization complete +[Apr 20, 2026 - 00:28:43 | INFO | fastgen.trainer:__init__:60 ] Initializing checkpointer... +[Apr 20, 2026 - 00:28:43 | SUCCESS | fastgen.trainer:__init__:65 ] Checkpointer initialized successfully +[Apr 20, 2026 - 00:28:43 | SUCCESS | __main__:main:33 ] Trainer initialized successfully +[Apr 20, 2026 - 00:28:43 | INFO | fastgen.trainer:run:77 ] Starting training +[Apr 20, 2026 - 00:28:43 | INFO | fastgen.trainer:run:80 ] Initializing callbacks and model ... +[Apr 20, 2026 - 00:28:43 | INFO | fastgen.utils.checkpointer:load:151 ] Loading model from /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_shift_5_4gpu_bs16_lr1e5_10000iter/checkpoints/0010000.pth +[Apr 20, 2026 - 00:28:48 | INFO | fastgen.utils.checkpointer:load:154 ] Loading the model_dict... +[Apr 20, 2026 - 00:29:00 | INFO | fastgen.utils.checkpointer:load:159 ] Model net, loading info: +[Apr 20, 2026 - 00:29:00 | INFO | fastgen.trainer:load_pretrained_ckpt:252 ] Loaded net model from net in /home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_shift_5_4gpu_bs16_lr1e5_10000iter/checkpoints/0010000.pth at iteration 10000 +[Apr 20, 2026 - 00:29:00 | INFO | fastgen.trainer:load_pretrained_ckpt:262 ] Setting resume_iter for model to 10000. +[Apr 20, 2026 - 00:29:00 | INFO | fastgen.trainer:run:95 ] Starting model.on_train_begin ... +[Apr 20, 2026 - 00:30:08 | INFO | fastgen.methods.model:on_train_begin:296 ] Teacher check: add_teacher_to_fsdp_dict=True, fsdp_dict keys=['net', 'fake_score', 'teacher'], teacher in fsdp_dict=True +[Apr 20, 2026 - 00:30:08 | INFO | fastgen.trainer:run:99 ] model.on_train_begin completed +[Apr 20, 2026 - 00:30:08 | INFO | fastgen.trainer:run:110 ] Wrapping model into fsdp .. +[Apr 20, 2026 - 00:30:08 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:102 ] Fully sharding model with 4 ranks... +[Apr 20, 2026 - 00:30:08 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'net' (1.42B params)... +[Apr 20, 2026 - 00:30:08 | INFO | fastgen.networks.OmniAvatar.network_causal:fully_shard:2166 ] CausalOmniAvatarWan: keeping manual gradient checkpointing (not using apply_fsdp_checkpointing due to KV cache dynamics) +[Apr 20, 2026 - 00:30:10 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[Apr 20, 2026 - 00:30:11 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped net in 3.0s +[Apr 20, 2026 - 00:30:11 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'fake_score' (1.60B params)... +[Apr 20, 2026 - 00:30:11 | INFO | fastgen.networks.OmniAvatar.network:fully_shard:774 ] OmniAvatarWan: keeping manual gradient checkpointing (checkpoint_wrapper incompatible with inter-block audio injection) +[Apr 20, 2026 - 00:30:13 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[Apr 20, 2026 - 00:30:13 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped fake_score in 2.8s +[Apr 20, 2026 - 00:30:13 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:132 ] Starting FSDP2 wrap for 'teacher' (14.29B params)... +[Apr 20, 2026 - 00:30:13 | INFO | fastgen.networks.OmniAvatar.network:fully_shard:774 ] OmniAvatarWan: keeping manual gradient checkpointing (checkpoint_wrapper incompatible with inter-block audio injection) +[Apr 20, 2026 - 00:30:27 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:177 ] Completed sharding +[Apr 20, 2026 - 00:30:27 | INFO | fastgen.utils.distributed.fsdp:model_to_fsdp:214 ] FSDP2 wrapped teacher in 13.5s +[Apr 20, 2026 - 00:30:27 | INFO | fastgen.trainer:run:118 ] FSDP wrapping completed +[Apr 20, 2026 - 00:30:27 | INFO | fastgen.callbacks.ema:on_model_init_end:64 ] EMA ema is not enabled, skipping callback. +[Apr 20, 2026 - 00:30:27 | INFO | fastgen.trainer:run:133 ] Auto-Resume Details: None +[Apr 20, 2026 - 00:30:27 | INFO | fastgen.utils.basic_utils:set_random_seed:144 ] Using random seed 0. +[Apr 20, 2026 - 00:30:27 | INFO | fastgen.trainer:run:165 ] Instantiating dataloader... +[Apr 20, 2026 - 00:30:35 | INFO | fastgen.callbacks.wandb:on_dataloader_init_end:362 ] SKIP_GT_VAL_UPLOAD=1 — skipping GT val video upload +[Apr 20, 2026 - 00:30:35 | INFO | fastgen.callbacks.gpu_stats:on_train_begin:57 ] every_n to measure gpus stats: 1 +[Apr 20, 2026 - 00:30:35 | INFO | fastgen.callbacks.train_profiler:on_train_begin:54 ] every_n to profile trainer: 1 +[Apr 20, 2026 - 00:30:35 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] model (OmniAvatarSelfForcingReDMD) has 1596.36 M trainable and 17325.46 M total params (logical). +[Apr 20, 2026 - 00:30:35 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 420.42 M trainable and 4364.06 M total params LOCAL on rank 0. +[Apr 20, 2026 - 00:30:35 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 420.14 M trainable and 4362.26 M total params LOCAL on rank 1. +[Apr 20, 2026 - 00:30:35 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 420.14 M trainable and 4362.26 M total params LOCAL on rank 2. +[Apr 20, 2026 - 00:30:35 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] model (OmniAvatarSelfForcingReDMD) has 420.14 M trainable and 4362.26 M total params LOCAL on rank 3. +[Apr 20, 2026 - 00:30:35 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] net (CausalOmniAvatarWan) has 1421.38 M trainable and 1421.38 M total params (logical). +[Apr 20, 2026 - 00:30:35 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 376.68 M trainable and 376.68 M total params LOCAL on rank 0. +[Apr 20, 2026 - 00:30:35 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 376.40 M trainable and 376.40 M total params LOCAL on rank 1. +[Apr 20, 2026 - 00:30:35 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 376.40 M trainable and 376.40 M total params LOCAL on rank 2. +[Apr 20, 2026 - 00:30:35 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] net (CausalOmniAvatarWan) has 376.40 M trainable and 376.40 M total params LOCAL on rank 3. +[Apr 20, 2026 - 00:30:35 | INFO | fastgen.callbacks.param_count:on_train_begin:79 ] fake_score (OmniAvatarWan) has 174.98 M trainable and 1596.36 M total params (logical). +[Apr 20, 2026 - 00:30:35 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 43.75 M trainable and 399.30 M total params LOCAL on rank 0. +[Apr 20, 2026 - 00:30:35 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 43.75 M trainable and 399.02 M total params LOCAL on rank 1. +[Apr 20, 2026 - 00:30:35 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 43.75 M trainable and 399.02 M total params LOCAL on rank 2. +[Apr 20, 2026 - 00:30:35 | INFO | fastgen.callbacks.param_count:on_train_begin:86 ] fake_score (OmniAvatarWan) has 43.75 M trainable and 399.02 M total params LOCAL on rank 3. +[Apr 20, 2026 - 00:30:35 | INFO | fastgen.trainer:run:174 ] iter_start: 0 +[Apr 20, 2026 - 00:31:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.612867 38.729275 38.491783 +peak_gpu_mem_gb 60.567137 60.567137 60.567137 +peak_gpu_mem_reserved_gb 62.634766 62.634766 62.634766 +util 93.500000 99.000000 89.000000 +[Apr 20, 2026 - 00:31:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 1 : data loading time 1.05 +[Apr 20, 2026 - 00:31:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 1 : avg forward pass time 13.57 +[Apr 20, 2026 - 00:31:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 1 : backward pass time 11.55 +[Apr 20, 2026 - 00:31:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 1 : optimizer step time 1.11 +[Apr 20, 2026 - 00:31:30 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 1-------------------- +[Apr 20, 2026 - 00:31:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0057 iter count: 1.0 +[Apr 20, 2026 - 00:31:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0057 iter count: 1.0 +[Apr 20, 2026 - 00:31:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 00:31:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 00:32:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.788426 38.904678 38.667164 +peak_gpu_mem_gb 62.354346 62.354346 62.354346 +peak_gpu_mem_reserved_gb 64.101562 64.101562 64.101562 +util 99.000000 100.000000 97.000000 +[Apr 20, 2026 - 00:32:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 2 : avg iteration time 50.36 seconds +[Apr 20, 2026 - 00:32:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 2 : data loading time 0.00 +[Apr 20, 2026 - 00:32:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 2 : avg forward pass time 12.80 +[Apr 20, 2026 - 00:32:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 2 : backward pass time 11.57 +[Apr 20, 2026 - 00:32:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 2 : optimizer step time 1.19 +[Apr 20, 2026 - 00:32:20 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 2-------------------- +[Apr 20, 2026 - 00:32:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0062 iter count: 1.0 +[Apr 20, 2026 - 00:32:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0062 iter count: 1.0 +[Apr 20, 2026 - 00:32:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 00:32:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 00:33:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.788486 38.904774 38.667324 +peak_gpu_mem_gb 62.354346 62.354346 62.354346 +peak_gpu_mem_reserved_gb 64.062500 64.062500 64.062500 +util 96.250000 99.000000 93.000000 +[Apr 20, 2026 - 00:33:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 3 : avg iteration time 55.75 seconds +[Apr 20, 2026 - 00:33:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 3 : data loading time 0.00 +[Apr 20, 2026 - 00:33:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 3 : avg forward pass time 15.74 +[Apr 20, 2026 - 00:33:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 3 : backward pass time 11.72 +[Apr 20, 2026 - 00:33:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 3 : optimizer step time 0.98 +[Apr 20, 2026 - 00:33:16 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 3-------------------- +[Apr 20, 2026 - 00:33:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0062 iter count: 1.0 +[Apr 20, 2026 - 00:33:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0062 iter count: 1.0 +[Apr 20, 2026 - 00:33:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 00:33:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 00:34:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.788871 38.905758 38.667370 +peak_gpu_mem_gb 62.354346 62.354346 62.354346 +peak_gpu_mem_reserved_gb 64.082031 64.082031 64.082031 +util 97.000000 100.000000 93.000000 +[Apr 20, 2026 - 00:34:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 4 : avg iteration time 56.22 seconds +[Apr 20, 2026 - 00:34:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 4 : data loading time 0.00 +[Apr 20, 2026 - 00:34:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 4 : avg forward pass time 15.41 +[Apr 20, 2026 - 00:34:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 4 : backward pass time 11.60 +[Apr 20, 2026 - 00:34:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 4 : optimizer step time 1.25 +[Apr 20, 2026 - 00:34:12 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 4-------------------- +[Apr 20, 2026 - 00:34:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0066 iter count: 1.0 +[Apr 20, 2026 - 00:34:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0066 iter count: 1.0 +[Apr 20, 2026 - 00:34:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 00:34:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 00:37:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.850340 38.968113 38.729740 +peak_gpu_mem_gb 126.538745 126.538745 126.538745 +peak_gpu_mem_reserved_gb 134.244141 138.126953 130.351562 +util 64.000000 68.000000 57.000000 +[Apr 20, 2026 - 00:37:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 5 : avg iteration time 197.49 seconds +[Apr 20, 2026 - 00:37:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 5 : data loading time 0.00 +[Apr 20, 2026 - 00:37:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 5 : avg forward pass time 89.44 +[Apr 20, 2026 - 00:37:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 5 : backward pass time 9.76 +[Apr 20, 2026 - 00:37:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 5 : optimizer step time 0.42 +[Apr 20, 2026 - 00:37:30 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 5-------------------- +[Apr 20, 2026 - 00:37:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 5960.1862 iter count: 1.0 +[Apr 20, 2026 - 00:37:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6960 iter count: 1.0 +[Apr 20, 2026 - 00:37:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 5960.1862iter count: 1.0 +[Apr 20, 2026 - 00:37:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 00:37:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 2.8835iter count: 1.0 +[Apr 20, 2026 - 00:37:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.4076 iter count: 1.0 +[Apr 20, 2026 - 00:37:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.0298 iter count: 1.0 +[Apr 20, 2026 - 00:37:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 7158.9399iter count: 1.0 +[Apr 20, 2026 - 00:37:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 2.2598 iter count: 1.0 +[Apr 20, 2026 - 00:37:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 172761.5469iter count: 1.0 +[Apr 20, 2026 - 00:37:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6960iter count: 1.0 +[Apr 20, 2026 - 00:37:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.4461 iter count: 1.0 +[Apr 20, 2026 - 00:37:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 984.6369iter count: 1.0 +[Apr 20, 2026 - 00:37:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 1.8823 iter count: 1.0 +[Apr 20, 2026 - 00:37:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 43.1497 iter count: 1.0 +[Apr 20, 2026 - 00:37:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.7531 iter count: 1.0 +[Apr 20, 2026 - 00:37:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 13443.8252iter count: 1.0 +[Apr 20, 2026 - 00:37:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 2.0492 iter count: 1.0 +[Apr 20, 2026 - 00:37:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 60.2434 iter count: 1.0 +[Apr 20, 2026 - 00:37:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0066 iter count: 1.0 +[Apr 20, 2026 - 00:37:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 00:38:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.850495 38.968140 38.729748 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.257324 67.378906 66.912109 +util 98.750000 100.000000 97.000000 +[Apr 20, 2026 - 00:38:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 6 : avg iteration time 50.43 seconds +[Apr 20, 2026 - 00:38:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 6 : data loading time 0.00 +[Apr 20, 2026 - 00:38:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 6 : avg forward pass time 13.17 +[Apr 20, 2026 - 00:38:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 6 : backward pass time 11.55 +[Apr 20, 2026 - 00:38:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 6 : optimizer step time 0.99 +[Apr 20, 2026 - 00:38:20 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 6-------------------- +[Apr 20, 2026 - 00:38:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0062 iter count: 1.0 +[Apr 20, 2026 - 00:38:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0062 iter count: 1.0 +[Apr 20, 2026 - 00:38:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 00:38:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 00:39:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.851088 38.968124 38.730629 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.252441 67.437500 66.990234 +util 97.000000 100.000000 91.000000 +[Apr 20, 2026 - 00:39:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 7 : avg iteration time 56.41 seconds +[Apr 20, 2026 - 00:39:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 7 : data loading time 0.00 +[Apr 20, 2026 - 00:39:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 7 : avg forward pass time 15.31 +[Apr 20, 2026 - 00:39:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 7 : backward pass time 11.75 +[Apr 20, 2026 - 00:39:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 7 : optimizer step time 1.50 +[Apr 20, 2026 - 00:39:17 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 7-------------------- +[Apr 20, 2026 - 00:39:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0062 iter count: 1.0 +[Apr 20, 2026 - 00:39:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0062 iter count: 1.0 +[Apr 20, 2026 - 00:39:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 00:39:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 00:40:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.851092 38.968102 38.730629 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 68.746582 70.308594 66.990234 +util 97.250000 100.000000 92.000000 +[Apr 20, 2026 - 00:40:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 8 : avg iteration time 55.51 seconds +[Apr 20, 2026 - 00:40:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 8 : data loading time 0.00 +[Apr 20, 2026 - 00:40:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 8 : avg forward pass time 15.70 +[Apr 20, 2026 - 00:40:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 8 : backward pass time 11.56 +[Apr 20, 2026 - 00:40:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 8 : optimizer step time 1.00 +[Apr 20, 2026 - 00:40:12 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 8-------------------- +[Apr 20, 2026 - 00:40:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0059 iter count: 1.0 +[Apr 20, 2026 - 00:40:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0059 iter count: 1.0 +[Apr 20, 2026 - 00:40:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 00:40:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 00:41:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.851038 38.968132 38.730644 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.252441 67.339844 66.990234 +util 99.000000 100.000000 97.000000 +[Apr 20, 2026 - 00:41:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 9 : avg iteration time 59.17 seconds +[Apr 20, 2026 - 00:41:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 9 : data loading time 0.00 +[Apr 20, 2026 - 00:41:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 9 : avg forward pass time 16.92 +[Apr 20, 2026 - 00:41:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 9 : backward pass time 12.09 +[Apr 20, 2026 - 00:41:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 9 : optimizer step time 1.56 +[Apr 20, 2026 - 00:41:11 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 9-------------------- +[Apr 20, 2026 - 00:41:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0057 iter count: 1.0 +[Apr 20, 2026 - 00:41:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0057 iter count: 1.0 +[Apr 20, 2026 - 00:41:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 00:41:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 00:44:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.851865 38.968586 38.731113 +peak_gpu_mem_gb 129.238753 129.240298 129.238238 +peak_gpu_mem_reserved_gb 133.133301 133.171875 133.076172 +util 67.750000 76.000000 62.000000 +[Apr 20, 2026 - 00:44:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 10 : avg iteration time 184.88 seconds +[Apr 20, 2026 - 00:44:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 10 : data loading time 0.00 +[Apr 20, 2026 - 00:44:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 10 : avg forward pass time 83.48 +[Apr 20, 2026 - 00:44:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 10 : backward pass time 8.34 +[Apr 20, 2026 - 00:44:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 10 : optimizer step time 0.34 +[Apr 20, 2026 - 00:44:16 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 10-------------------- +[Apr 20, 2026 - 00:44:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 25607.4414 iter count: 1.0 +[Apr 20, 2026 - 00:44:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.7275 iter count: 1.0 +[Apr 20, 2026 - 00:44:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 25607.4414iter count: 1.0 +[Apr 20, 2026 - 00:44:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 00:44:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.6207iter count: 1.0 +[Apr 20, 2026 - 00:44:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.7761 iter count: 1.0 +[Apr 20, 2026 - 00:44:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.7249 iter count: 1.0 +[Apr 20, 2026 - 00:44:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 30472.9688iter count: 1.0 +[Apr 20, 2026 - 00:44:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 4.7215 iter count: 1.0 +[Apr 20, 2026 - 00:44:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 693666.1875iter count: 1.0 +[Apr 20, 2026 - 00:44:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.7275iter count: 1.0 +[Apr 20, 2026 - 00:44:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 6.7249 iter count: 1.0 +[Apr 20, 2026 - 00:44:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 693666.1875iter count: 1.0 +[Apr 20, 2026 - 00:44:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 2.1050 iter count: 1.0 +[Apr 20, 2026 - 00:44:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 67.3537 iter count: 1.0 +[Apr 20, 2026 - 00:44:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 3.7874 iter count: 1.0 +[Apr 20, 2026 - 00:44:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 1948.4640iter count: 1.0 +[Apr 20, 2026 - 00:44:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.9310 iter count: 1.0 +[Apr 20, 2026 - 00:44:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 19186.7754iter count: 1.0 +[Apr 20, 2026 - 00:44:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0058 iter count: 1.0 +[Apr 20, 2026 - 00:44:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 00:45:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.851872 38.968582 38.731110 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.179199 67.378906 66.929688 +util 94.750000 99.000000 91.000000 +[Apr 20, 2026 - 00:45:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 11 : avg iteration time 57.98 seconds +[Apr 20, 2026 - 00:45:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 11 : data loading time 0.00 +[Apr 20, 2026 - 00:45:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 11 : avg forward pass time 16.40 +[Apr 20, 2026 - 00:45:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 11 : backward pass time 12.62 +[Apr 20, 2026 - 00:45:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 11 : optimizer step time 1.01 +[Apr 20, 2026 - 00:45:14 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 11-------------------- +[Apr 20, 2026 - 00:45:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0071 iter count: 1.0 +[Apr 20, 2026 - 00:45:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0071 iter count: 1.0 +[Apr 20, 2026 - 00:45:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 00:45:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 00:46:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.851886 38.968582 38.731129 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.369629 67.398438 67.302734 +util 96.500000 100.000000 93.000000 +[Apr 20, 2026 - 00:46:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 12 : avg iteration time 52.23 seconds +[Apr 20, 2026 - 00:46:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 12 : data loading time 0.00 +[Apr 20, 2026 - 00:46:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 12 : avg forward pass time 14.06 +[Apr 20, 2026 - 00:46:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 12 : backward pass time 11.54 +[Apr 20, 2026 - 00:46:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 12 : optimizer step time 0.99 +[Apr 20, 2026 - 00:46:06 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 12-------------------- +[Apr 20, 2026 - 00:46:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0057 iter count: 1.0 +[Apr 20, 2026 - 00:46:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0057 iter count: 1.0 +[Apr 20, 2026 - 00:46:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 00:46:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 00:46:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.851879 38.968590 38.731110 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.296387 67.398438 67.029297 +util 95.750000 99.000000 92.000000 +[Apr 20, 2026 - 00:46:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 13 : avg iteration time 47.62 seconds +[Apr 20, 2026 - 00:46:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 13 : data loading time 0.00 +[Apr 20, 2026 - 00:46:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 13 : avg forward pass time 11.20 +[Apr 20, 2026 - 00:46:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 13 : backward pass time 11.58 +[Apr 20, 2026 - 00:46:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 13 : optimizer step time 0.97 +[Apr 20, 2026 - 00:46:54 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 13-------------------- +[Apr 20, 2026 - 00:46:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0057 iter count: 1.0 +[Apr 20, 2026 - 00:46:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0057 iter count: 1.0 +[Apr 20, 2026 - 00:46:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 00:46:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 00:47:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.851889 38.968613 38.731113 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.296387 67.378906 67.126953 +util 96.000000 99.000000 93.000000 +[Apr 20, 2026 - 00:47:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 14 : avg iteration time 56.20 seconds +[Apr 20, 2026 - 00:47:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 14 : data loading time 0.00 +[Apr 20, 2026 - 00:47:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 14 : avg forward pass time 15.74 +[Apr 20, 2026 - 00:47:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 14 : backward pass time 12.20 +[Apr 20, 2026 - 00:47:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 14 : optimizer step time 0.97 +[Apr 20, 2026 - 00:47:50 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 14-------------------- +[Apr 20, 2026 - 00:47:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0061 iter count: 1.0 +[Apr 20, 2026 - 00:47:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0061 iter count: 1.0 +[Apr 20, 2026 - 00:47:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 00:47:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 00:50:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.852572 38.968609 38.732887 +peak_gpu_mem_gb 129.238753 129.240298 129.238238 +peak_gpu_mem_reserved_gb 133.104004 133.132812 133.056641 +util 69.500000 76.000000 64.000000 +[Apr 20, 2026 - 00:50:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 15 : avg iteration time 182.66 seconds +[Apr 20, 2026 - 00:50:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 15 : data loading time 0.00 +[Apr 20, 2026 - 00:50:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 15 : avg forward pass time 82.61 +[Apr 20, 2026 - 00:50:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 15 : backward pass time 8.40 +[Apr 20, 2026 - 00:50:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 15 : optimizer step time 0.38 +[Apr 20, 2026 - 00:50:53 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 15-------------------- +[Apr 20, 2026 - 00:50:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 8425.7834 iter count: 1.0 +[Apr 20, 2026 - 00:50:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6766 iter count: 1.0 +[Apr 20, 2026 - 00:50:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 8425.7834iter count: 1.0 +[Apr 20, 2026 - 00:50:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 00:50:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 2.9170iter count: 1.0 +[Apr 20, 2026 - 00:50:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.3309 iter count: 1.0 +[Apr 20, 2026 - 00:50:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.2771 iter count: 1.0 +[Apr 20, 2026 - 00:50:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 10414.9219iter count: 1.0 +[Apr 20, 2026 - 00:50:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 1.9383 iter count: 1.0 +[Apr 20, 2026 - 00:50:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 283306.0938iter count: 1.0 +[Apr 20, 2026 - 00:50:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6766iter count: 1.0 +[Apr 20, 2026 - 00:50:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.0657 iter count: 1.0 +[Apr 20, 2026 - 00:50:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 3399.2998iter count: 1.0 +[Apr 20, 2026 - 00:50:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.4429 iter count: 1.0 +[Apr 20, 2026 - 00:50:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 978.3767iter count: 1.0 +[Apr 20, 2026 - 00:50:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 2.0535 iter count: 1.0 +[Apr 20, 2026 - 00:50:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 60.7638 iter count: 1.0 +[Apr 20, 2026 - 00:50:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 2.6070 iter count: 1.0 +[Apr 20, 2026 - 00:50:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 183.8333iter count: 1.0 +[Apr 20, 2026 - 00:50:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0056 iter count: 1.0 +[Apr 20, 2026 - 00:50:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 00:51:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.852803 38.968773 38.732910 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.232910 67.320312 66.970703 +util 97.500000 100.000000 92.000000 +[Apr 20, 2026 - 00:51:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 16 : avg iteration time 58.01 seconds +[Apr 20, 2026 - 00:51:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 16 : data loading time 0.00 +[Apr 20, 2026 - 00:51:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 16 : avg forward pass time 16.51 +[Apr 20, 2026 - 00:51:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 16 : backward pass time 12.41 +[Apr 20, 2026 - 00:51:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 16 : optimizer step time 1.01 +[Apr 20, 2026 - 00:51:51 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 16-------------------- +[Apr 20, 2026 - 00:51:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0053 iter count: 1.0 +[Apr 20, 2026 - 00:51:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0053 iter count: 1.0 +[Apr 20, 2026 - 00:51:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 00:51:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 00:52:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.853401 38.968773 38.732918 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.198730 67.320312 67.029297 +util 96.000000 99.000000 93.000000 +[Apr 20, 2026 - 00:52:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 17 : avg iteration time 52.92 seconds +[Apr 20, 2026 - 00:52:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 17 : data loading time 0.00 +[Apr 20, 2026 - 00:52:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 17 : avg forward pass time 14.21 +[Apr 20, 2026 - 00:52:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 17 : backward pass time 11.58 +[Apr 20, 2026 - 00:52:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 17 : optimizer step time 0.98 +[Apr 20, 2026 - 00:52:44 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 17-------------------- +[Apr 20, 2026 - 00:52:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0053 iter count: 1.0 +[Apr 20, 2026 - 00:52:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0053 iter count: 1.0 +[Apr 20, 2026 - 00:52:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 00:52:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 00:53:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.853621 38.968845 38.732895 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.135254 67.183594 67.009766 +util 96.000000 99.000000 93.000000 +[Apr 20, 2026 - 00:53:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 18 : avg iteration time 61.80 seconds +[Apr 20, 2026 - 00:53:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 18 : data loading time 0.00 +[Apr 20, 2026 - 00:53:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 18 : avg forward pass time 18.27 +[Apr 20, 2026 - 00:53:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 18 : backward pass time 12.65 +[Apr 20, 2026 - 00:53:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 18 : optimizer step time 1.03 +[Apr 20, 2026 - 00:53:46 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 18-------------------- +[Apr 20, 2026 - 00:53:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0068 iter count: 1.0 +[Apr 20, 2026 - 00:53:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0068 iter count: 1.0 +[Apr 20, 2026 - 00:53:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 00:53:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 00:54:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.853512 38.968845 38.732895 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 68.634277 70.171875 67.009766 +util 97.000000 100.000000 91.000000 +[Apr 20, 2026 - 00:54:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 19 : avg iteration time 49.86 seconds +[Apr 20, 2026 - 00:54:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 19 : data loading time 0.00 +[Apr 20, 2026 - 00:54:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 19 : avg forward pass time 12.88 +[Apr 20, 2026 - 00:54:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 19 : backward pass time 11.58 +[Apr 20, 2026 - 00:54:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 19 : optimizer step time 0.98 +[Apr 20, 2026 - 00:54:35 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 19-------------------- +[Apr 20, 2026 - 00:54:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0053 iter count: 1.0 +[Apr 20, 2026 - 00:54:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0053 iter count: 1.0 +[Apr 20, 2026 - 00:54:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 00:54:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 00:57:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.853722 38.969002 38.733025 +peak_gpu_mem_gb 129.238753 129.240298 129.238238 +peak_gpu_mem_reserved_gb 133.143066 133.230469 133.035156 +util 72.000000 80.000000 63.000000 +[Apr 20, 2026 - 00:57:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 20 : avg iteration time 187.69 seconds +[Apr 20, 2026 - 00:57:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 20 : data loading time 0.00 +[Apr 20, 2026 - 00:57:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 20 : avg forward pass time 84.90 +[Apr 20, 2026 - 00:57:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 20 : backward pass time 8.38 +[Apr 20, 2026 - 00:57:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 20 : optimizer step time 0.33 +[Apr 20, 2026 - 00:57:43 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 20-------------------- +[Apr 20, 2026 - 00:57:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 10046.5526 iter count: 1.0 +[Apr 20, 2026 - 00:57:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.7449 iter count: 1.0 +[Apr 20, 2026 - 00:57:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 10046.5526iter count: 1.0 +[Apr 20, 2026 - 00:57:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 00:57:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.1013iter count: 1.0 +[Apr 20, 2026 - 00:57:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.7237 iter count: 1.0 +[Apr 20, 2026 - 00:57:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.1103 iter count: 1.0 +[Apr 20, 2026 - 00:57:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 12102.2217iter count: 1.0 +[Apr 20, 2026 - 00:57:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 4.2521 iter count: 1.0 +[Apr 20, 2026 - 00:57:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 202908.1250iter count: 1.0 +[Apr 20, 2026 - 00:57:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.7449iter count: 1.0 +[Apr 20, 2026 - 00:57:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.9111 iter count: 1.0 +[Apr 20, 2026 - 00:57:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 2495.2771iter count: 1.0 +[Apr 20, 2026 - 00:57:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 1.5415 iter count: 1.0 +[Apr 20, 2026 - 00:57:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 21.8247 iter count: 1.0 +[Apr 20, 2026 - 00:57:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 3.1420 iter count: 1.0 +[Apr 20, 2026 - 00:57:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 535.9355iter count: 1.0 +[Apr 20, 2026 - 00:57:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 5.5193 iter count: 1.0 +[Apr 20, 2026 - 00:57:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 62225.8984iter count: 1.0 +[Apr 20, 2026 - 00:57:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0051 iter count: 1.0 +[Apr 20, 2026 - 00:57:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 00:58:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.853778 38.969002 38.733025 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.286621 67.437500 66.912109 +util 96.250000 99.000000 90.000000 +[Apr 20, 2026 - 00:58:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 21 : avg iteration time 58.79 seconds +[Apr 20, 2026 - 00:58:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 21 : data loading time 0.00 +[Apr 20, 2026 - 00:58:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 21 : avg forward pass time 16.90 +[Apr 20, 2026 - 00:58:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 21 : backward pass time 11.51 +[Apr 20, 2026 - 00:58:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 21 : optimizer step time 1.00 +[Apr 20, 2026 - 00:58:42 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 21-------------------- +[Apr 20, 2026 - 00:58:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0048 iter count: 1.0 +[Apr 20, 2026 - 00:58:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0048 iter count: 1.0 +[Apr 20, 2026 - 00:58:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 00:58:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 00:59:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.853778 38.969002 38.733025 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.174316 67.359375 66.970703 +util 98.250000 100.000000 94.000000 +[Apr 20, 2026 - 00:59:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 22 : avg iteration time 61.97 seconds +[Apr 20, 2026 - 00:59:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 22 : data loading time 0.00 +[Apr 20, 2026 - 00:59:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 22 : avg forward pass time 17.98 +[Apr 20, 2026 - 00:59:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 22 : backward pass time 12.41 +[Apr 20, 2026 - 00:59:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 22 : optimizer step time 1.60 +[Apr 20, 2026 - 00:59:44 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 22-------------------- +[Apr 20, 2026 - 00:59:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0057 iter count: 1.0 +[Apr 20, 2026 - 00:59:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0057 iter count: 1.0 +[Apr 20, 2026 - 00:59:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 00:59:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 01:00:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.853778 38.969002 38.733025 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.267090 67.378906 66.970703 +util 97.750000 99.000000 94.000000 +[Apr 20, 2026 - 01:00:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 23 : avg iteration time 47.81 seconds +[Apr 20, 2026 - 01:00:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 23 : data loading time 0.00 +[Apr 20, 2026 - 01:00:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 23 : avg forward pass time 11.14 +[Apr 20, 2026 - 01:00:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 23 : backward pass time 12.84 +[Apr 20, 2026 - 01:00:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 23 : optimizer step time 1.08 +[Apr 20, 2026 - 01:00:32 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 23-------------------- +[Apr 20, 2026 - 01:00:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0060 iter count: 1.0 +[Apr 20, 2026 - 01:00:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0060 iter count: 1.0 +[Apr 20, 2026 - 01:00:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 01:00:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 01:01:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.853786 38.969006 38.733028 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.193848 67.378906 66.990234 +util 97.250000 100.000000 91.000000 +[Apr 20, 2026 - 01:01:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 24 : avg iteration time 55.25 seconds +[Apr 20, 2026 - 01:01:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 24 : data loading time 0.00 +[Apr 20, 2026 - 01:01:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 24 : avg forward pass time 15.46 +[Apr 20, 2026 - 01:01:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 24 : backward pass time 11.53 +[Apr 20, 2026 - 01:01:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 24 : optimizer step time 0.99 +[Apr 20, 2026 - 01:01:27 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 24-------------------- +[Apr 20, 2026 - 01:01:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0056 iter count: 1.0 +[Apr 20, 2026 - 01:01:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0056 iter count: 1.0 +[Apr 20, 2026 - 01:01:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 01:01:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 01:04:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.853429 38.969040 38.733082 +peak_gpu_mem_gb 129.238753 129.240298 129.238238 +peak_gpu_mem_reserved_gb 133.133301 133.171875 133.035156 +util 71.500000 78.000000 67.000000 +[Apr 20, 2026 - 01:04:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 25 : avg iteration time 187.72 seconds +[Apr 20, 2026 - 01:04:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 25 : data loading time 0.00 +[Apr 20, 2026 - 01:04:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 25 : avg forward pass time 85.13 +[Apr 20, 2026 - 01:04:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 25 : backward pass time 8.42 +[Apr 20, 2026 - 01:04:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 25 : optimizer step time 0.34 +[Apr 20, 2026 - 01:04:35 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 25-------------------- +[Apr 20, 2026 - 01:04:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 12721.5561 iter count: 1.0 +[Apr 20, 2026 - 01:04:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.7416 iter count: 1.0 +[Apr 20, 2026 - 01:04:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 12721.5561iter count: 1.0 +[Apr 20, 2026 - 01:04:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 01:04:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.0292iter count: 1.0 +[Apr 20, 2026 - 01:04:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.3715 iter count: 1.0 +[Apr 20, 2026 - 01:04:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.4281 iter count: 1.0 +[Apr 20, 2026 - 01:04:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 27641.5098iter count: 1.0 +[Apr 20, 2026 - 01:04:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 2.1020 iter count: 1.0 +[Apr 20, 2026 - 01:04:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 383155.9375iter count: 1.0 +[Apr 20, 2026 - 01:04:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.7416iter count: 1.0 +[Apr 20, 2026 - 01:04:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 2.1849 iter count: 1.0 +[Apr 20, 2026 - 01:04:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 79.0342 iter count: 1.0 +[Apr 20, 2026 - 01:04:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 0.7512 iter count: 1.0 +[Apr 20, 2026 - 01:04:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 4.4920 iter count: 1.0 +[Apr 20, 2026 - 01:04:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 3.9754 iter count: 1.0 +[Apr 20, 2026 - 01:04:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 2837.6655iter count: 1.0 +[Apr 20, 2026 - 01:04:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 5.9044 iter count: 1.0 +[Apr 20, 2026 - 01:04:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 134431.4062iter count: 1.0 +[Apr 20, 2026 - 01:04:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0054 iter count: 1.0 +[Apr 20, 2026 - 01:04:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 01:05:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.853486 38.969040 38.733097 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.262207 67.398438 66.931641 +util 96.500000 99.000000 90.000000 +[Apr 20, 2026 - 01:05:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 26 : avg iteration time 59.64 seconds +[Apr 20, 2026 - 01:05:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 26 : data loading time 0.00 +[Apr 20, 2026 - 01:05:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 26 : avg forward pass time 17.59 +[Apr 20, 2026 - 01:05:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 26 : backward pass time 11.52 +[Apr 20, 2026 - 01:05:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 26 : optimizer step time 1.05 +[Apr 20, 2026 - 01:05:34 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 26-------------------- +[Apr 20, 2026 - 01:05:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0069 iter count: 1.0 +[Apr 20, 2026 - 01:05:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0069 iter count: 1.0 +[Apr 20, 2026 - 01:05:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 01:05:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 01:06:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.853852 38.969021 38.733086 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.276855 67.398438 66.931641 +util 98.250000 99.000000 96.000000 +[Apr 20, 2026 - 01:06:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 27 : avg iteration time 60.94 seconds +[Apr 20, 2026 - 01:06:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 27 : data loading time 0.00 +[Apr 20, 2026 - 01:06:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 27 : avg forward pass time 18.38 +[Apr 20, 2026 - 01:06:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 27 : backward pass time 11.54 +[Apr 20, 2026 - 01:06:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 27 : optimizer step time 0.99 +[Apr 20, 2026 - 01:06:35 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 27-------------------- +[Apr 20, 2026 - 01:06:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0051 iter count: 1.0 +[Apr 20, 2026 - 01:06:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0051 iter count: 1.0 +[Apr 20, 2026 - 01:06:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 01:06:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 01:07:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.853858 38.969021 38.733086 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.257324 67.359375 66.951172 +util 94.250000 99.000000 90.000000 +[Apr 20, 2026 - 01:07:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 28 : avg iteration time 50.35 seconds +[Apr 20, 2026 - 01:07:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 28 : data loading time 0.00 +[Apr 20, 2026 - 01:07:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 28 : avg forward pass time 12.49 +[Apr 20, 2026 - 01:07:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 28 : backward pass time 11.54 +[Apr 20, 2026 - 01:07:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 28 : optimizer step time 0.97 +[Apr 20, 2026 - 01:07:26 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 28-------------------- +[Apr 20, 2026 - 01:07:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0056 iter count: 1.0 +[Apr 20, 2026 - 01:07:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0056 iter count: 1.0 +[Apr 20, 2026 - 01:07:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 01:07:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 01:08:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.853989 38.969028 38.733112 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.252441 67.398438 66.931641 +util 97.750000 100.000000 92.000000 +[Apr 20, 2026 - 01:08:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 29 : avg iteration time 54.09 seconds +[Apr 20, 2026 - 01:08:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 29 : data loading time 0.00 +[Apr 20, 2026 - 01:08:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 29 : avg forward pass time 14.15 +[Apr 20, 2026 - 01:08:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 29 : backward pass time 12.51 +[Apr 20, 2026 - 01:08:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 29 : optimizer step time 1.59 +[Apr 20, 2026 - 01:08:20 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 29-------------------- +[Apr 20, 2026 - 01:08:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0060 iter count: 1.0 +[Apr 20, 2026 - 01:08:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0060 iter count: 1.0 +[Apr 20, 2026 - 01:08:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 01:08:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 01:11:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.853705 38.969521 38.733505 +peak_gpu_mem_gb 129.238753 129.240298 129.238238 +peak_gpu_mem_reserved_gb 133.091309 133.117188 133.076172 +util 70.250000 77.000000 66.000000 +[Apr 20, 2026 - 01:11:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 30 : avg iteration time 190.20 seconds +[Apr 20, 2026 - 01:11:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 30 : data loading time 0.00 +[Apr 20, 2026 - 01:11:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 30 : avg forward pass time 86.56 +[Apr 20, 2026 - 01:11:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 30 : backward pass time 8.38 +[Apr 20, 2026 - 01:11:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 30 : optimizer step time 0.36 +[Apr 20, 2026 - 01:11:30 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 30-------------------- +[Apr 20, 2026 - 01:11:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 147806.4687 iter count: 1.0 +[Apr 20, 2026 - 01:11:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.7438 iter count: 1.0 +[Apr 20, 2026 - 01:11:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 147806.4687iter count: 1.0 +[Apr 20, 2026 - 01:11:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 01:11:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.6512iter count: 1.0 +[Apr 20, 2026 - 01:11:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.0973 iter count: 1.0 +[Apr 20, 2026 - 01:11:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.7823 iter count: 1.0 +[Apr 20, 2026 - 01:11:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 205276.3438iter count: 1.0 +[Apr 20, 2026 - 01:11:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 8.9761 iter count: 1.0 +[Apr 20, 2026 - 01:11:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 5749787.5000iter count: 1.0 +[Apr 20, 2026 - 01:11:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.7438iter count: 1.0 +[Apr 20, 2026 - 01:11:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.8524 iter count: 1.0 +[Apr 20, 2026 - 01:11:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 16396.1465iter count: 1.0 +[Apr 20, 2026 - 01:11:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.7377 iter count: 1.0 +[Apr 20, 2026 - 01:11:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 1764.1962iter count: 1.0 +[Apr 20, 2026 - 01:11:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 2.4938 iter count: 1.0 +[Apr 20, 2026 - 01:11:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 146.5985iter count: 1.0 +[Apr 20, 2026 - 01:11:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.2118 iter count: 1.0 +[Apr 20, 2026 - 01:11:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 4553.1406iter count: 1.0 +[Apr 20, 2026 - 01:11:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0056 iter count: 1.0 +[Apr 20, 2026 - 01:11:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 01:12:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.853831 38.969574 38.733521 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.584473 68.142578 67.320312 +util 97.750000 100.000000 93.000000 +[Apr 20, 2026 - 01:12:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 31 : avg iteration time 49.81 seconds +[Apr 20, 2026 - 01:12:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 31 : data loading time 0.00 +[Apr 20, 2026 - 01:12:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 31 : avg forward pass time 12.09 +[Apr 20, 2026 - 01:12:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 31 : backward pass time 12.76 +[Apr 20, 2026 - 01:12:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 31 : optimizer step time 1.33 +[Apr 20, 2026 - 01:12:20 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 31-------------------- +[Apr 20, 2026 - 01:12:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0055 iter count: 1.0 +[Apr 20, 2026 - 01:12:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0055 iter count: 1.0 +[Apr 20, 2026 - 01:12:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 01:12:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 01:13:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.854095 38.969589 38.733570 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 69.752441 70.269531 68.259766 +util 96.250000 99.000000 93.000000 +[Apr 20, 2026 - 01:13:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 32 : avg iteration time 50.13 seconds +[Apr 20, 2026 - 01:13:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 32 : data loading time 0.00 +[Apr 20, 2026 - 01:13:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 32 : avg forward pass time 12.93 +[Apr 20, 2026 - 01:13:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 32 : backward pass time 11.69 +[Apr 20, 2026 - 01:13:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 32 : optimizer step time 0.99 +[Apr 20, 2026 - 01:13:10 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 32-------------------- +[Apr 20, 2026 - 01:13:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0056 iter count: 1.0 +[Apr 20, 2026 - 01:13:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0056 iter count: 1.0 +[Apr 20, 2026 - 01:13:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 01:13:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 01:14:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.854147 38.969570 38.733574 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.950684 69.800781 67.126953 +util 96.250000 99.000000 93.000000 +[Apr 20, 2026 - 01:14:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 33 : avg iteration time 56.00 seconds +[Apr 20, 2026 - 01:14:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 33 : data loading time 0.00 +[Apr 20, 2026 - 01:14:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 33 : avg forward pass time 15.46 +[Apr 20, 2026 - 01:14:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 33 : backward pass time 11.55 +[Apr 20, 2026 - 01:14:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 33 : optimizer step time 0.97 +[Apr 20, 2026 - 01:14:06 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 33-------------------- +[Apr 20, 2026 - 01:14:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0049 iter count: 1.0 +[Apr 20, 2026 - 01:14:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0049 iter count: 1.0 +[Apr 20, 2026 - 01:14:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 01:14:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 01:15:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.854270 38.969570 38.733574 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 68.751465 70.308594 67.068359 +util 98.750000 100.000000 96.000000 +[Apr 20, 2026 - 01:15:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 34 : avg iteration time 53.82 seconds +[Apr 20, 2026 - 01:15:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 34 : data loading time 0.00 +[Apr 20, 2026 - 01:15:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 34 : avg forward pass time 14.02 +[Apr 20, 2026 - 01:15:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 34 : backward pass time 12.06 +[Apr 20, 2026 - 01:15:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 34 : optimizer step time 1.55 +[Apr 20, 2026 - 01:15:00 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 34-------------------- +[Apr 20, 2026 - 01:15:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0049 iter count: 1.0 +[Apr 20, 2026 - 01:15:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0049 iter count: 1.0 +[Apr 20, 2026 - 01:15:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 01:15:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 01:18:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.853985 38.969551 38.733528 +peak_gpu_mem_gb 129.238753 129.240298 129.238238 +peak_gpu_mem_reserved_gb 133.177246 133.210938 133.113281 +util 70.250000 74.000000 64.000000 +[Apr 20, 2026 - 01:18:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 35 : avg iteration time 187.05 seconds +[Apr 20, 2026 - 01:18:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 35 : data loading time 0.00 +[Apr 20, 2026 - 01:18:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 35 : avg forward pass time 84.71 +[Apr 20, 2026 - 01:18:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 35 : backward pass time 8.37 +[Apr 20, 2026 - 01:18:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 35 : optimizer step time 0.34 +[Apr 20, 2026 - 01:18:07 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 35-------------------- +[Apr 20, 2026 - 01:18:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 25772.2146 iter count: 1.0 +[Apr 20, 2026 - 01:18:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.7695 iter count: 1.0 +[Apr 20, 2026 - 01:18:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 25772.2146iter count: 1.0 +[Apr 20, 2026 - 01:18:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 01:18:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.7052iter count: 1.0 +[Apr 20, 2026 - 01:18:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.6977 iter count: 1.0 +[Apr 20, 2026 - 01:18:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.3444 iter count: 1.0 +[Apr 20, 2026 - 01:18:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 33321.6992iter count: 1.0 +[Apr 20, 2026 - 01:18:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 4.0363 iter count: 1.0 +[Apr 20, 2026 - 01:18:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 324078.0938iter count: 1.0 +[Apr 20, 2026 - 01:18:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.7695iter count: 1.0 +[Apr 20, 2026 - 01:18:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 2.9957 iter count: 1.0 +[Apr 20, 2026 - 01:18:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 399.9657iter count: 1.0 +[Apr 20, 2026 - 01:18:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 0.8291 iter count: 1.0 +[Apr 20, 2026 - 01:18:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 5.2499 iter count: 1.0 +[Apr 20, 2026 - 01:18:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.6997 iter count: 1.0 +[Apr 20, 2026 - 01:18:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 12081.5303iter count: 1.0 +[Apr 20, 2026 - 01:18:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 2.1045 iter count: 1.0 +[Apr 20, 2026 - 01:18:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 67.2880 iter count: 1.0 +[Apr 20, 2026 - 01:18:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0054 iter count: 1.0 +[Apr 20, 2026 - 01:18:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 01:18:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.854116 38.969551 38.733551 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.340332 67.476562 67.009766 +util 98.500000 100.000000 96.000000 +[Apr 20, 2026 - 01:18:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 36 : avg iteration time 52.35 seconds +[Apr 20, 2026 - 01:18:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 36 : data loading time 0.00 +[Apr 20, 2026 - 01:18:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 36 : avg forward pass time 13.58 +[Apr 20, 2026 - 01:18:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 36 : backward pass time 12.08 +[Apr 20, 2026 - 01:18:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 36 : optimizer step time 1.56 +[Apr 20, 2026 - 01:18:59 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 36-------------------- +[Apr 20, 2026 - 01:18:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0061 iter count: 1.0 +[Apr 20, 2026 - 01:18:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0061 iter count: 1.0 +[Apr 20, 2026 - 01:18:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 01:18:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 01:19:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.854268 38.969551 38.733551 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.496582 67.945312 67.205078 +util 98.250000 100.000000 94.000000 +[Apr 20, 2026 - 01:19:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 37 : avg iteration time 60.46 seconds +[Apr 20, 2026 - 01:19:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 37 : data loading time 0.00 +[Apr 20, 2026 - 01:19:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 37 : avg forward pass time 18.19 +[Apr 20, 2026 - 01:19:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 37 : backward pass time 11.56 +[Apr 20, 2026 - 01:19:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 37 : optimizer step time 0.99 +[Apr 20, 2026 - 01:19:59 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 37-------------------- +[Apr 20, 2026 - 01:19:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0058 iter count: 1.0 +[Apr 20, 2026 - 01:19:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0058 iter count: 1.0 +[Apr 20, 2026 - 01:19:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 01:19:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 01:20:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.854259 38.969559 38.733528 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.457520 67.925781 67.029297 +util 98.000000 100.000000 93.000000 +[Apr 20, 2026 - 01:20:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 38 : avg iteration time 56.77 seconds +[Apr 20, 2026 - 01:20:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 38 : data loading time 0.00 +[Apr 20, 2026 - 01:20:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 38 : avg forward pass time 15.48 +[Apr 20, 2026 - 01:20:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 38 : backward pass time 12.51 +[Apr 20, 2026 - 01:20:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 38 : optimizer step time 1.59 +[Apr 20, 2026 - 01:20:56 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 38-------------------- +[Apr 20, 2026 - 01:20:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0055 iter count: 1.0 +[Apr 20, 2026 - 01:20:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0055 iter count: 1.0 +[Apr 20, 2026 - 01:20:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 01:20:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 01:21:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.854268 38.969570 38.733528 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.325684 67.437500 66.990234 +util 98.500000 100.000000 96.000000 +[Apr 20, 2026 - 01:21:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 39 : avg iteration time 58.17 seconds +[Apr 20, 2026 - 01:21:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 39 : data loading time 0.00 +[Apr 20, 2026 - 01:21:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 39 : avg forward pass time 17.04 +[Apr 20, 2026 - 01:21:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 39 : backward pass time 11.54 +[Apr 20, 2026 - 01:21:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 39 : optimizer step time 0.98 +[Apr 20, 2026 - 01:21:54 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 39-------------------- +[Apr 20, 2026 - 01:21:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0071 iter count: 1.0 +[Apr 20, 2026 - 01:21:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0071 iter count: 1.0 +[Apr 20, 2026 - 01:21:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 01:21:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 01:24:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.854228 38.969551 38.733543 +peak_gpu_mem_gb 129.238753 129.240298 129.238238 +peak_gpu_mem_reserved_gb 133.133301 133.171875 133.054688 +util 73.250000 77.000000 67.000000 +[Apr 20, 2026 - 01:24:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 40 : avg iteration time 184.47 seconds +[Apr 20, 2026 - 01:24:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 40 : data loading time 0.00 +[Apr 20, 2026 - 01:24:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 40 : avg forward pass time 83.69 +[Apr 20, 2026 - 01:24:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 40 : backward pass time 8.41 +[Apr 20, 2026 - 01:24:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 40 : optimizer step time 0.34 +[Apr 20, 2026 - 01:24:59 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 40-------------------- +[Apr 20, 2026 - 01:24:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 74366.2192 iter count: 1.0 +[Apr 20, 2026 - 01:24:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6585 iter count: 1.0 +[Apr 20, 2026 - 01:24:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 74366.2192iter count: 1.0 +[Apr 20, 2026 - 01:24:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 01:24:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.5915iter count: 1.0 +[Apr 20, 2026 - 01:24:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.6800 iter count: 1.0 +[Apr 20, 2026 - 01:24:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.2219 iter count: 1.0 +[Apr 20, 2026 - 01:24:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 84208.9062iter count: 1.0 +[Apr 20, 2026 - 01:24:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 3.8964 iter count: 1.0 +[Apr 20, 2026 - 01:24:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 1874580.3750iter count: 1.0 +[Apr 20, 2026 - 01:24:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6585iter count: 1.0 +[Apr 20, 2026 - 01:24:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.3826 iter count: 1.0 +[Apr 20, 2026 - 01:24:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 867.2104iter count: 1.0 +[Apr 20, 2026 - 01:24:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 2.9817 iter count: 1.0 +[Apr 20, 2026 - 01:24:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 388.9584iter count: 1.0 +[Apr 20, 2026 - 01:24:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 1.6684 iter count: 1.0 +[Apr 20, 2026 - 01:24:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 28.1280 iter count: 1.0 +[Apr 20, 2026 - 01:24:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 3.2700 iter count: 1.0 +[Apr 20, 2026 - 01:24:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 692.2463iter count: 1.0 +[Apr 20, 2026 - 01:24:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0064 iter count: 1.0 +[Apr 20, 2026 - 01:24:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 01:26:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.854224 38.969555 38.733528 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.096191 67.339844 66.912109 +util 95.750000 99.000000 92.000000 +[Apr 20, 2026 - 01:26:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 41 : avg iteration time 63.05 seconds +[Apr 20, 2026 - 01:26:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 41 : data loading time 0.00 +[Apr 20, 2026 - 01:26:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 41 : avg forward pass time 18.85 +[Apr 20, 2026 - 01:26:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 41 : backward pass time 11.57 +[Apr 20, 2026 - 01:26:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 41 : optimizer step time 0.98 +[Apr 20, 2026 - 01:26:02 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 41-------------------- +[Apr 20, 2026 - 01:26:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0063 iter count: 1.0 +[Apr 20, 2026 - 01:26:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0063 iter count: 1.0 +[Apr 20, 2026 - 01:26:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 01:26:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 01:26:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.854413 38.969555 38.733528 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.091309 67.144531 67.048828 +util 96.250000 99.000000 93.000000 +[Apr 20, 2026 - 01:26:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 42 : avg iteration time 54.04 seconds +[Apr 20, 2026 - 01:26:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 42 : data loading time 0.00 +[Apr 20, 2026 - 01:26:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 42 : avg forward pass time 14.30 +[Apr 20, 2026 - 01:26:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 42 : backward pass time 12.81 +[Apr 20, 2026 - 01:26:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 42 : optimizer step time 1.07 +[Apr 20, 2026 - 01:26:56 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 42-------------------- +[Apr 20, 2026 - 01:26:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0057 iter count: 1.0 +[Apr 20, 2026 - 01:26:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0057 iter count: 1.0 +[Apr 20, 2026 - 01:26:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 01:26:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 01:27:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.854411 38.969555 38.733536 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.042480 67.144531 67.007812 +util 96.750000 99.000000 91.000000 +[Apr 20, 2026 - 01:27:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 43 : avg iteration time 49.34 seconds +[Apr 20, 2026 - 01:27:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 43 : data loading time 0.00 +[Apr 20, 2026 - 01:27:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 43 : avg forward pass time 12.63 +[Apr 20, 2026 - 01:27:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 43 : backward pass time 11.54 +[Apr 20, 2026 - 01:27:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 43 : optimizer step time 0.98 +[Apr 20, 2026 - 01:27:45 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 43-------------------- +[Apr 20, 2026 - 01:27:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0058 iter count: 1.0 +[Apr 20, 2026 - 01:27:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0058 iter count: 1.0 +[Apr 20, 2026 - 01:27:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 01:27:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 01:28:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.854324 38.969597 38.733536 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.037598 67.144531 66.990234 +util 97.750000 100.000000 93.000000 +[Apr 20, 2026 - 01:28:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 44 : avg iteration time 62.15 seconds +[Apr 20, 2026 - 01:28:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 44 : data loading time 0.00 +[Apr 20, 2026 - 01:28:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 44 : avg forward pass time 18.25 +[Apr 20, 2026 - 01:28:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 44 : backward pass time 12.69 +[Apr 20, 2026 - 01:28:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 44 : optimizer step time 1.35 +[Apr 20, 2026 - 01:28:47 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 44-------------------- +[Apr 20, 2026 - 01:28:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0055 iter count: 1.0 +[Apr 20, 2026 - 01:28:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0055 iter count: 1.0 +[Apr 20, 2026 - 01:28:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 01:28:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 01:31:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.854177 38.969563 38.733540 +peak_gpu_mem_gb 129.238753 129.240298 129.238238 +peak_gpu_mem_reserved_gb 133.090332 133.095703 133.076172 +util 68.000000 75.000000 62.000000 +[Apr 20, 2026 - 01:31:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 45 : avg iteration time 175.16 seconds +[Apr 20, 2026 - 01:31:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 45 : data loading time 0.00 +[Apr 20, 2026 - 01:31:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 45 : avg forward pass time 78.60 +[Apr 20, 2026 - 01:31:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 45 : backward pass time 8.46 +[Apr 20, 2026 - 01:31:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 45 : optimizer step time 0.38 +[Apr 20, 2026 - 01:31:43 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 45-------------------- +[Apr 20, 2026 - 01:31:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 130418.4646 iter count: 1.0 +[Apr 20, 2026 - 01:31:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.7593 iter count: 1.0 +[Apr 20, 2026 - 01:31:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 130418.4646iter count: 1.0 +[Apr 20, 2026 - 01:31:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 01:31:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.1692iter count: 1.0 +[Apr 20, 2026 - 01:31:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.9041 iter count: 1.0 +[Apr 20, 2026 - 01:31:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.7738 iter count: 1.0 +[Apr 20, 2026 - 01:31:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 205081.1562iter count: 1.0 +[Apr 20, 2026 - 01:31:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 45.0662iter count: 1.0 +[Apr 20, 2026 - 01:31:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 5652580.5000iter count: 1.0 +[Apr 20, 2026 - 01:31:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.7593iter count: 1.0 +[Apr 20, 2026 - 01:31:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 5.7663 iter count: 1.0 +[Apr 20, 2026 - 01:31:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 101988.7344iter count: 1.0 +[Apr 20, 2026 - 01:31:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 2.8227 iter count: 1.0 +[Apr 20, 2026 - 01:31:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 282.9713iter count: 1.0 +[Apr 20, 2026 - 01:31:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 5.5955 iter count: 1.0 +[Apr 20, 2026 - 01:31:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 72481.1641iter count: 1.0 +[Apr 20, 2026 - 01:31:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.4172 iter count: 1.0 +[Apr 20, 2026 - 01:31:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 6867.1152iter count: 1.0 +[Apr 20, 2026 - 01:31:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0054 iter count: 1.0 +[Apr 20, 2026 - 01:31:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 01:32:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.854310 38.969566 38.733540 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.452637 67.496094 67.380859 +util 96.250000 99.000000 93.000000 +[Apr 20, 2026 - 01:32:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 46 : avg iteration time 50.76 seconds +[Apr 20, 2026 - 01:32:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 46 : data loading time 0.00 +[Apr 20, 2026 - 01:32:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 46 : avg forward pass time 13.34 +[Apr 20, 2026 - 01:32:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 46 : backward pass time 11.55 +[Apr 20, 2026 - 01:32:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 46 : optimizer step time 0.97 +[Apr 20, 2026 - 01:32:33 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 46-------------------- +[Apr 20, 2026 - 01:32:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0060 iter count: 1.0 +[Apr 20, 2026 - 01:32:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0060 iter count: 1.0 +[Apr 20, 2026 - 01:32:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 01:32:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 01:33:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.854381 38.969566 38.733562 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.384277 67.476562 67.244141 +util 99.250000 100.000000 98.000000 +[Apr 20, 2026 - 01:33:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 47 : avg iteration time 57.05 seconds +[Apr 20, 2026 - 01:33:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 47 : data loading time 0.00 +[Apr 20, 2026 - 01:33:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 47 : avg forward pass time 15.68 +[Apr 20, 2026 - 01:33:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 47 : backward pass time 12.90 +[Apr 20, 2026 - 01:33:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 47 : optimizer step time 1.23 +[Apr 20, 2026 - 01:33:30 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 47-------------------- +[Apr 20, 2026 - 01:33:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0059 iter count: 1.0 +[Apr 20, 2026 - 01:33:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0059 iter count: 1.0 +[Apr 20, 2026 - 01:33:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 01:33:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 01:34:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.854383 38.969566 38.733574 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.340332 67.417969 67.164062 +util 94.750000 100.000000 91.000000 +[Apr 20, 2026 - 01:34:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 48 : avg iteration time 55.49 seconds +[Apr 20, 2026 - 01:34:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 48 : data loading time 0.00 +[Apr 20, 2026 - 01:34:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 48 : avg forward pass time 15.70 +[Apr 20, 2026 - 01:34:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 48 : backward pass time 11.53 +[Apr 20, 2026 - 01:34:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 48 : optimizer step time 0.99 +[Apr 20, 2026 - 01:34:26 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 48-------------------- +[Apr 20, 2026 - 01:34:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0073 iter count: 1.0 +[Apr 20, 2026 - 01:34:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0073 iter count: 1.0 +[Apr 20, 2026 - 01:34:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 01:34:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 01:35:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.854454 38.969589 38.733555 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 68.170410 70.269531 67.419922 +util 97.000000 100.000000 91.000000 +[Apr 20, 2026 - 01:35:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 49 : avg iteration time 62.20 seconds +[Apr 20, 2026 - 01:35:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 49 : data loading time 0.00 +[Apr 20, 2026 - 01:35:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 49 : avg forward pass time 18.30 +[Apr 20, 2026 - 01:35:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 49 : backward pass time 12.83 +[Apr 20, 2026 - 01:35:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 49 : optimizer step time 1.22 +[Apr 20, 2026 - 01:35:28 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 49-------------------- +[Apr 20, 2026 - 01:35:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0054 iter count: 1.0 +[Apr 20, 2026 - 01:35:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0054 iter count: 1.0 +[Apr 20, 2026 - 01:35:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 01:35:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 01:38:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.854094 38.969582 38.733562 +peak_gpu_mem_gb 129.238753 129.240298 129.238238 +peak_gpu_mem_reserved_gb 133.167480 133.230469 133.095703 +util 67.250000 73.000000 63.000000 +[Apr 20, 2026 - 01:38:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 50 : avg iteration time 182.05 seconds +[Apr 20, 2026 - 01:38:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 50 : data loading time 0.00 +[Apr 20, 2026 - 01:38:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 50 : avg forward pass time 82.24 +[Apr 20, 2026 - 01:38:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 50 : backward pass time 8.35 +[Apr 20, 2026 - 01:38:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 50 : optimizer step time 0.33 +[Apr 20, 2026 - 01:38:30 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 50-------------------- +[Apr 20, 2026 - 01:38:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 60664.5580 iter count: 1.0 +[Apr 20, 2026 - 01:38:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.7360 iter count: 1.0 +[Apr 20, 2026 - 01:38:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 60664.5580iter count: 1.0 +[Apr 20, 2026 - 01:38:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 01:38:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.4248iter count: 1.0 +[Apr 20, 2026 - 01:38:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.2724 iter count: 1.0 +[Apr 20, 2026 - 01:38:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.8809 iter count: 1.0 +[Apr 20, 2026 - 01:38:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 74352.9062iter count: 1.0 +[Apr 20, 2026 - 01:38:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 12.7419iter count: 1.0 +[Apr 20, 2026 - 01:38:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 947631.1875iter count: 1.0 +[Apr 20, 2026 - 01:38:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.7360iter count: 1.0 +[Apr 20, 2026 - 01:38:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 5.8270 iter count: 1.0 +[Apr 20, 2026 - 01:38:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 115153.3828iter count: 1.0 +[Apr 20, 2026 - 01:38:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 2.5016 iter count: 1.0 +[Apr 20, 2026 - 01:38:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 148.8759iter count: 1.0 +[Apr 20, 2026 - 01:38:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 1.8343 iter count: 1.0 +[Apr 20, 2026 - 01:38:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 39.1972 iter count: 1.0 +[Apr 20, 2026 - 01:38:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 2.6928 iter count: 1.0 +[Apr 20, 2026 - 01:38:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 218.2238iter count: 1.0 +[Apr 20, 2026 - 01:38:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0060 iter count: 1.0 +[Apr 20, 2026 - 01:38:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 01:39:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.854111 38.969627 38.733601 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.228027 67.496094 66.931641 +util 96.750000 99.000000 94.000000 +[Apr 20, 2026 - 01:39:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 51 : avg iteration time 61.22 seconds +[Apr 20, 2026 - 01:39:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 51 : data loading time 0.00 +[Apr 20, 2026 - 01:39:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 51 : avg forward pass time 17.78 +[Apr 20, 2026 - 01:39:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 51 : backward pass time 12.97 +[Apr 20, 2026 - 01:39:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 51 : optimizer step time 1.12 +[Apr 20, 2026 - 01:39:31 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 51-------------------- +[Apr 20, 2026 - 01:39:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0063 iter count: 1.0 +[Apr 20, 2026 - 01:39:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0063 iter count: 1.0 +[Apr 20, 2026 - 01:39:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 01:39:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 01:40:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.854426 38.969627 38.733601 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.198730 67.398438 66.970703 +util 96.250000 100.000000 92.000000 +[Apr 20, 2026 - 01:40:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 52 : avg iteration time 55.35 seconds +[Apr 20, 2026 - 01:40:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 52 : data loading time 0.00 +[Apr 20, 2026 - 01:40:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 52 : avg forward pass time 15.64 +[Apr 20, 2026 - 01:40:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 52 : backward pass time 11.54 +[Apr 20, 2026 - 01:40:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 52 : optimizer step time 0.98 +[Apr 20, 2026 - 01:40:27 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 52-------------------- +[Apr 20, 2026 - 01:40:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0075 iter count: 1.0 +[Apr 20, 2026 - 01:40:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0075 iter count: 1.0 +[Apr 20, 2026 - 01:40:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 01:40:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 01:41:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.854609 38.969627 38.733601 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.223145 67.417969 67.009766 +util 97.250000 100.000000 92.000000 +[Apr 20, 2026 - 01:41:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 53 : avg iteration time 62.50 seconds +[Apr 20, 2026 - 01:41:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 53 : data loading time 0.00 +[Apr 20, 2026 - 01:41:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 53 : avg forward pass time 18.18 +[Apr 20, 2026 - 01:41:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 53 : backward pass time 12.87 +[Apr 20, 2026 - 01:41:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 53 : optimizer step time 1.45 +[Apr 20, 2026 - 01:41:29 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 53-------------------- +[Apr 20, 2026 - 01:41:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0069 iter count: 1.0 +[Apr 20, 2026 - 01:41:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0069 iter count: 1.0 +[Apr 20, 2026 - 01:41:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 01:41:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 01:42:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.854243 38.969582 38.733578 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 68.683105 70.347656 67.009766 +util 97.000000 100.000000 92.000000 +[Apr 20, 2026 - 01:42:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 54 : avg iteration time 60.19 seconds +[Apr 20, 2026 - 01:42:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 54 : data loading time 0.00 +[Apr 20, 2026 - 01:42:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 54 : avg forward pass time 18.05 +[Apr 20, 2026 - 01:42:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 54 : backward pass time 11.57 +[Apr 20, 2026 - 01:42:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 54 : optimizer step time 0.98 +[Apr 20, 2026 - 01:42:29 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 54-------------------- +[Apr 20, 2026 - 01:42:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0070 iter count: 1.0 +[Apr 20, 2026 - 01:42:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0070 iter count: 1.0 +[Apr 20, 2026 - 01:42:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 01:42:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 01:45:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.854203 38.969582 38.733593 +peak_gpu_mem_gb 129.238753 129.240298 129.238238 +peak_gpu_mem_reserved_gb 133.084473 133.093750 133.074219 +util 72.500000 78.000000 68.000000 +[Apr 20, 2026 - 01:45:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 55 : avg iteration time 169.20 seconds +[Apr 20, 2026 - 01:45:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 55 : data loading time 0.00 +[Apr 20, 2026 - 01:45:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 55 : avg forward pass time 75.79 +[Apr 20, 2026 - 01:45:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 55 : backward pass time 8.42 +[Apr 20, 2026 - 01:45:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 55 : optimizer step time 0.33 +[Apr 20, 2026 - 01:45:19 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 55-------------------- +[Apr 20, 2026 - 01:45:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 12438.8194 iter count: 1.0 +[Apr 20, 2026 - 01:45:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6216 iter count: 1.0 +[Apr 20, 2026 - 01:45:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 12438.8194iter count: 1.0 +[Apr 20, 2026 - 01:45:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 01:45:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.3648iter count: 1.0 +[Apr 20, 2026 - 01:45:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.8321 iter count: 1.0 +[Apr 20, 2026 - 01:45:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.1458 iter count: 1.0 +[Apr 20, 2026 - 01:45:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 20520.1992iter count: 1.0 +[Apr 20, 2026 - 01:45:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 5.2810 iter count: 1.0 +[Apr 20, 2026 - 01:45:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 217867.6719iter count: 1.0 +[Apr 20, 2026 - 01:45:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6216iter count: 1.0 +[Apr 20, 2026 - 01:45:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 6.0048 iter count: 1.0 +[Apr 20, 2026 - 01:45:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 164327.8438iter count: 1.0 +[Apr 20, 2026 - 01:45:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 2.2416 iter count: 1.0 +[Apr 20, 2026 - 01:45:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 88.5122 iter count: 1.0 +[Apr 20, 2026 - 01:45:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.0889 iter count: 1.0 +[Apr 20, 2026 - 01:45:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 3560.8188iter count: 1.0 +[Apr 20, 2026 - 01:45:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 3.4436 iter count: 1.0 +[Apr 20, 2026 - 01:45:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 979.6866iter count: 1.0 +[Apr 20, 2026 - 01:45:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0058 iter count: 1.0 +[Apr 20, 2026 - 01:45:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 01:46:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.854268 38.969627 38.733555 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.267090 67.339844 67.087891 +util 98.250000 100.000000 95.000000 +[Apr 20, 2026 - 01:46:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 56 : avg iteration time 57.25 seconds +[Apr 20, 2026 - 01:46:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 56 : data loading time 0.00 +[Apr 20, 2026 - 01:46:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 56 : avg forward pass time 16.57 +[Apr 20, 2026 - 01:46:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 56 : backward pass time 11.54 +[Apr 20, 2026 - 01:46:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 56 : optimizer step time 1.02 +[Apr 20, 2026 - 01:46:16 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 56-------------------- +[Apr 20, 2026 - 01:46:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0082 iter count: 1.0 +[Apr 20, 2026 - 01:46:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[Apr 20, 2026 - 01:46:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 01:46:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 01:47:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.854386 38.969604 38.733555 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.340332 67.400391 67.300781 +util 98.500000 100.000000 96.000000 +[Apr 20, 2026 - 01:47:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 57 : avg iteration time 56.21 seconds +[Apr 20, 2026 - 01:47:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 57 : data loading time 0.00 +[Apr 20, 2026 - 01:47:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 57 : avg forward pass time 15.37 +[Apr 20, 2026 - 01:47:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 57 : backward pass time 11.55 +[Apr 20, 2026 - 01:47:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 57 : optimizer step time 0.97 +[Apr 20, 2026 - 01:47:12 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 57-------------------- +[Apr 20, 2026 - 01:47:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0057 iter count: 1.0 +[Apr 20, 2026 - 01:47:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0057 iter count: 1.0 +[Apr 20, 2026 - 01:47:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 01:47:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 01:48:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.854445 38.969582 38.733555 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.335449 67.419922 67.300781 +util 98.250000 99.000000 96.000000 +[Apr 20, 2026 - 01:48:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 58 : avg iteration time 59.32 seconds +[Apr 20, 2026 - 01:48:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 58 : data loading time 0.00 +[Apr 20, 2026 - 01:48:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 58 : avg forward pass time 17.39 +[Apr 20, 2026 - 01:48:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 58 : backward pass time 12.00 +[Apr 20, 2026 - 01:48:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 58 : optimizer step time 0.98 +[Apr 20, 2026 - 01:48:11 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 58-------------------- +[Apr 20, 2026 - 01:48:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0077 iter count: 1.0 +[Apr 20, 2026 - 01:48:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0077 iter count: 1.0 +[Apr 20, 2026 - 01:48:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 01:48:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 01:49:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.854301 38.969589 38.733555 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.301270 67.439453 67.164062 +util 95.500000 99.000000 92.000000 +[Apr 20, 2026 - 01:49:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 59 : avg iteration time 57.76 seconds +[Apr 20, 2026 - 01:49:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 59 : data loading time 0.00 +[Apr 20, 2026 - 01:49:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 59 : avg forward pass time 16.71 +[Apr 20, 2026 - 01:49:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 59 : backward pass time 11.50 +[Apr 20, 2026 - 01:49:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 59 : optimizer step time 1.02 +[Apr 20, 2026 - 01:49:09 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 59-------------------- +[Apr 20, 2026 - 01:49:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0073 iter count: 1.0 +[Apr 20, 2026 - 01:49:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0073 iter count: 1.0 +[Apr 20, 2026 - 01:49:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 01:49:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 01:52:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.854127 38.969608 38.733570 +peak_gpu_mem_gb 129.238753 129.240298 129.238238 +peak_gpu_mem_reserved_gb 133.124023 133.136719 133.093750 +util 68.000000 74.000000 64.000000 +[Apr 20, 2026 - 01:52:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 60 : avg iteration time 187.82 seconds +[Apr 20, 2026 - 01:52:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 60 : data loading time 0.00 +[Apr 20, 2026 - 01:52:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 60 : avg forward pass time 84.79 +[Apr 20, 2026 - 01:52:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 60 : backward pass time 8.39 +[Apr 20, 2026 - 01:52:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 60 : optimizer step time 0.33 +[Apr 20, 2026 - 01:52:17 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 60-------------------- +[Apr 20, 2026 - 01:52:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 5705.5526 iter count: 1.0 +[Apr 20, 2026 - 01:52:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6953 iter count: 1.0 +[Apr 20, 2026 - 01:52:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 5705.5526iter count: 1.0 +[Apr 20, 2026 - 01:52:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 01:52:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.2484iter count: 1.0 +[Apr 20, 2026 - 01:52:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.9763 iter count: 1.0 +[Apr 20, 2026 - 01:52:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 5.6978 iter count: 1.0 +[Apr 20, 2026 - 01:52:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 7739.2588iter count: 1.0 +[Apr 20, 2026 - 01:52:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 7.0470 iter count: 1.0 +[Apr 20, 2026 - 01:52:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 88925.9453iter count: 1.0 +[Apr 20, 2026 - 01:52:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6953iter count: 1.0 +[Apr 20, 2026 - 01:52:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 2.8826 iter count: 1.0 +[Apr 20, 2026 - 01:52:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 319.0325iter count: 1.0 +[Apr 20, 2026 - 01:52:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.4285 iter count: 1.0 +[Apr 20, 2026 - 01:52:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 7023.7251iter count: 1.0 +[Apr 20, 2026 - 01:52:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 2.5353 iter count: 1.0 +[Apr 20, 2026 - 01:52:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 159.2745iter count: 1.0 +[Apr 20, 2026 - 01:52:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 3.7914 iter count: 1.0 +[Apr 20, 2026 - 01:52:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 1964.2292iter count: 1.0 +[Apr 20, 2026 - 01:52:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0080 iter count: 1.0 +[Apr 20, 2026 - 01:52:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 01:53:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.85425 38.969608 38.733570 +peak_gpu_mem_gb 65.49830 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.23291 67.359375 66.931641 +util 98.25000 100.000000 95.000000 +[Apr 20, 2026 - 01:53:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 61 : avg iteration time 54.91 seconds +[Apr 20, 2026 - 01:53:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 61 : data loading time 0.00 +[Apr 20, 2026 - 01:53:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 61 : avg forward pass time 14.64 +[Apr 20, 2026 - 01:53:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 61 : backward pass time 11.59 +[Apr 20, 2026 - 01:53:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 61 : optimizer step time 0.97 +[Apr 20, 2026 - 01:53:12 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 61-------------------- +[Apr 20, 2026 - 01:53:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0056 iter count: 1.0 +[Apr 20, 2026 - 01:53:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0056 iter count: 1.0 +[Apr 20, 2026 - 01:53:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 01:53:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 01:54:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.854249 38.969608 38.733570 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.237793 67.378906 66.931641 +util 96.500000 100.000000 93.000000 +[Apr 20, 2026 - 01:54:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 62 : avg iteration time 57.10 seconds +[Apr 20, 2026 - 01:54:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 62 : data loading time 0.00 +[Apr 20, 2026 - 01:54:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 62 : avg forward pass time 15.57 +[Apr 20, 2026 - 01:54:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 62 : backward pass time 12.91 +[Apr 20, 2026 - 01:54:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 62 : optimizer step time 1.47 +[Apr 20, 2026 - 01:54:09 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 62-------------------- +[Apr 20, 2026 - 01:54:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0063 iter count: 1.0 +[Apr 20, 2026 - 01:54:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0063 iter count: 1.0 +[Apr 20, 2026 - 01:54:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 01:54:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 01:54:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.854313 38.969608 38.733570 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.267090 67.359375 67.068359 +util 97.750000 100.000000 92.000000 +[Apr 20, 2026 - 01:54:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 63 : avg iteration time 50.47 seconds +[Apr 20, 2026 - 01:54:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 63 : data loading time 0.00 +[Apr 20, 2026 - 01:54:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 63 : avg forward pass time 12.93 +[Apr 20, 2026 - 01:54:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 63 : backward pass time 12.08 +[Apr 20, 2026 - 01:54:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 63 : optimizer step time 0.97 +[Apr 20, 2026 - 01:54:59 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 63-------------------- +[Apr 20, 2026 - 01:54:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0057 iter count: 1.0 +[Apr 20, 2026 - 01:54:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0057 iter count: 1.0 +[Apr 20, 2026 - 01:54:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 01:54:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 01:55:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.854312 38.969608 38.733570 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.242676 67.398438 66.970703 +util 95.750000 99.000000 92.000000 +[Apr 20, 2026 - 01:55:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 64 : avg iteration time 55.73 seconds +[Apr 20, 2026 - 01:55:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 64 : data loading time 0.00 +[Apr 20, 2026 - 01:55:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 64 : avg forward pass time 15.50 +[Apr 20, 2026 - 01:55:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 64 : backward pass time 11.58 +[Apr 20, 2026 - 01:55:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 64 : optimizer step time 0.99 +[Apr 20, 2026 - 01:55:55 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 64-------------------- +[Apr 20, 2026 - 01:55:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0068 iter count: 1.0 +[Apr 20, 2026 - 01:55:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0068 iter count: 1.0 +[Apr 20, 2026 - 01:55:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 01:55:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 01:59:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.853962 38.969608 38.733570 +peak_gpu_mem_gb 129.238753 129.240298 129.238238 +peak_gpu_mem_reserved_gb 133.109375 133.175781 133.035156 +util 66.250000 73.000000 62.000000 +[Apr 20, 2026 - 01:59:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 65 : avg iteration time 189.19 seconds +[Apr 20, 2026 - 01:59:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 65 : data loading time 0.00 +[Apr 20, 2026 - 01:59:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 65 : avg forward pass time 85.53 +[Apr 20, 2026 - 01:59:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 65 : backward pass time 8.42 +[Apr 20, 2026 - 01:59:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 65 : optimizer step time 0.33 +[Apr 20, 2026 - 01:59:04 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 65-------------------- +[Apr 20, 2026 - 01:59:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 86768.9138 iter count: 1.0 +[Apr 20, 2026 - 01:59:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.7009 iter count: 1.0 +[Apr 20, 2026 - 01:59:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 86768.9138iter count: 1.0 +[Apr 20, 2026 - 01:59:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 01:59:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.9799iter count: 1.0 +[Apr 20, 2026 - 01:59:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.4222 iter count: 1.0 +[Apr 20, 2026 - 01:59:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.3266 iter count: 1.0 +[Apr 20, 2026 - 01:59:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 105946.6875iter count: 1.0 +[Apr 20, 2026 - 01:59:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 17.1905iter count: 1.0 +[Apr 20, 2026 - 01:59:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 2310961.7500iter count: 1.0 +[Apr 20, 2026 - 01:59:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.7009iter count: 1.0 +[Apr 20, 2026 - 01:59:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.5792 iter count: 1.0 +[Apr 20, 2026 - 01:59:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 1284.8212iter count: 1.0 +[Apr 20, 2026 - 01:59:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.1188 iter count: 1.0 +[Apr 20, 2026 - 01:59:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 511.6732iter count: 1.0 +[Apr 20, 2026 - 01:59:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.0766 iter count: 1.0 +[Apr 20, 2026 - 01:59:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 3474.2683iter count: 1.0 +[Apr 20, 2026 - 01:59:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 3.5249 iter count: 1.0 +[Apr 20, 2026 - 01:59:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 1152.6447iter count: 1.0 +[Apr 20, 2026 - 01:59:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0059 iter count: 1.0 +[Apr 20, 2026 - 01:59:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 02:00:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.854026 38.969608 38.733570 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.193848 67.359375 66.990234 +util 93.750000 99.000000 89.000000 +[Apr 20, 2026 - 02:00:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 66 : avg iteration time 66.18 seconds +[Apr 20, 2026 - 02:00:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 66 : data loading time 0.00 +[Apr 20, 2026 - 02:00:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 66 : avg forward pass time 20.35 +[Apr 20, 2026 - 02:00:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 66 : backward pass time 11.52 +[Apr 20, 2026 - 02:00:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 66 : optimizer step time 1.04 +[Apr 20, 2026 - 02:00:11 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 66-------------------- +[Apr 20, 2026 - 02:00:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0088 iter count: 1.0 +[Apr 20, 2026 - 02:00:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[Apr 20, 2026 - 02:00:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 02:00:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 02:01:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.854395 38.969608 38.733570 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.218262 67.398438 66.970703 +util 98.250000 100.000000 96.000000 +[Apr 20, 2026 - 02:01:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 67 : avg iteration time 60.88 seconds +[Apr 20, 2026 - 02:01:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 67 : data loading time 0.00 +[Apr 20, 2026 - 02:01:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 67 : avg forward pass time 18.39 +[Apr 20, 2026 - 02:01:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 67 : backward pass time 11.55 +[Apr 20, 2026 - 02:01:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 67 : optimizer step time 0.98 +[Apr 20, 2026 - 02:01:11 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 67-------------------- +[Apr 20, 2026 - 02:01:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0079 iter count: 1.0 +[Apr 20, 2026 - 02:01:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0079 iter count: 1.0 +[Apr 20, 2026 - 02:01:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 02:01:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 02:02:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.854469 38.969627 38.733570 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.320801 67.400391 67.164062 +util 97.750000 100.000000 93.000000 +[Apr 20, 2026 - 02:02:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 68 : avg iteration time 59.51 seconds +[Apr 20, 2026 - 02:02:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 68 : data loading time 0.00 +[Apr 20, 2026 - 02:02:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 68 : avg forward pass time 16.86 +[Apr 20, 2026 - 02:02:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 68 : backward pass time 12.36 +[Apr 20, 2026 - 02:02:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 68 : optimizer step time 1.57 +[Apr 20, 2026 - 02:02:11 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 68-------------------- +[Apr 20, 2026 - 02:02:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0060 iter count: 1.0 +[Apr 20, 2026 - 02:02:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0060 iter count: 1.0 +[Apr 20, 2026 - 02:02:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 02:02:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 02:03:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.854389 38.969608 38.733570 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.140137 67.203125 67.046875 +util 98.750000 100.000000 96.000000 +[Apr 20, 2026 - 02:03:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 69 : avg iteration time 55.56 seconds +[Apr 20, 2026 - 02:03:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 69 : data loading time 0.00 +[Apr 20, 2026 - 02:03:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 69 : avg forward pass time 15.73 +[Apr 20, 2026 - 02:03:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 69 : backward pass time 11.53 +[Apr 20, 2026 - 02:03:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 69 : optimizer step time 1.00 +[Apr 20, 2026 - 02:03:07 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 69-------------------- +[Apr 20, 2026 - 02:03:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0066 iter count: 1.0 +[Apr 20, 2026 - 02:03:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0066 iter count: 1.0 +[Apr 20, 2026 - 02:03:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 02:03:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 02:06:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.854381 38.969608 38.733593 +peak_gpu_mem_gb 129.238753 129.240298 129.238238 +peak_gpu_mem_reserved_gb 133.138184 133.154297 133.132812 +util 71.000000 75.000000 64.000000 +[Apr 20, 2026 - 02:06:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 70 : avg iteration time 193.18 seconds +[Apr 20, 2026 - 02:06:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 70 : data loading time 0.00 +[Apr 20, 2026 - 02:06:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 70 : avg forward pass time 87.63 +[Apr 20, 2026 - 02:06:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 70 : backward pass time 8.40 +[Apr 20, 2026 - 02:06:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 70 : optimizer step time 0.33 +[Apr 20, 2026 - 02:06:20 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 70-------------------- +[Apr 20, 2026 - 02:06:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 90513.9041 iter count: 1.0 +[Apr 20, 2026 - 02:06:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6707 iter count: 1.0 +[Apr 20, 2026 - 02:06:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 90513.9041iter count: 1.0 +[Apr 20, 2026 - 02:06:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 02:06:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.0056iter count: 1.0 +[Apr 20, 2026 - 02:06:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.0141 iter count: 1.0 +[Apr 20, 2026 - 02:06:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.4568 iter count: 1.0 +[Apr 20, 2026 - 02:06:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 193943.5000iter count: 1.0 +[Apr 20, 2026 - 02:06:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 7.5997 iter count: 1.0 +[Apr 20, 2026 - 02:06:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 2998649.2500iter count: 1.0 +[Apr 20, 2026 - 02:06:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6707iter count: 1.0 +[Apr 20, 2026 - 02:06:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 2.8094 iter count: 1.0 +[Apr 20, 2026 - 02:06:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 275.5388iter count: 1.0 +[Apr 20, 2026 - 02:06:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 2.8938 iter count: 1.0 +[Apr 20, 2026 - 02:06:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 326.2265iter count: 1.0 +[Apr 20, 2026 - 02:06:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 1.3512 iter count: 1.0 +[Apr 20, 2026 - 02:06:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 14.9142 iter count: 1.0 +[Apr 20, 2026 - 02:06:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 3.2882 iter count: 1.0 +[Apr 20, 2026 - 02:06:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 717.9410iter count: 1.0 +[Apr 20, 2026 - 02:06:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0055 iter count: 1.0 +[Apr 20, 2026 - 02:06:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 02:07:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.854386 38.969631 38.733570 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.325684 67.457031 67.029297 +util 97.000000 100.000000 91.000000 +[Apr 20, 2026 - 02:07:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 71 : avg iteration time 61.69 seconds +[Apr 20, 2026 - 02:07:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 71 : data loading time 0.00 +[Apr 20, 2026 - 02:07:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 71 : avg forward pass time 17.92 +[Apr 20, 2026 - 02:07:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 71 : backward pass time 13.08 +[Apr 20, 2026 - 02:07:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 71 : optimizer step time 1.22 +[Apr 20, 2026 - 02:07:21 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 71-------------------- +[Apr 20, 2026 - 02:07:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0060 iter count: 1.0 +[Apr 20, 2026 - 02:07:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0060 iter count: 1.0 +[Apr 20, 2026 - 02:07:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 02:07:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 02:08:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.854386 38.969608 38.733570 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.086426 67.359375 66.910156 +util 97.500000 100.000000 92.000000 +[Apr 20, 2026 - 02:08:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 72 : avg iteration time 47.69 seconds +[Apr 20, 2026 - 02:08:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 72 : data loading time 0.00 +[Apr 20, 2026 - 02:08:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 72 : avg forward pass time 11.40 +[Apr 20, 2026 - 02:08:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 72 : backward pass time 12.31 +[Apr 20, 2026 - 02:08:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 72 : optimizer step time 1.00 +[Apr 20, 2026 - 02:08:09 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 72-------------------- +[Apr 20, 2026 - 02:08:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0062 iter count: 1.0 +[Apr 20, 2026 - 02:08:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0062 iter count: 1.0 +[Apr 20, 2026 - 02:08:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 02:08:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 02:09:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.854445 38.969608 38.733570 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.086426 67.339844 66.929688 +util 98.250000 99.000000 96.000000 +[Apr 20, 2026 - 02:09:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 73 : avg iteration time 55.25 seconds +[Apr 20, 2026 - 02:09:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 73 : data loading time 0.00 +[Apr 20, 2026 - 02:09:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 73 : avg forward pass time 15.38 +[Apr 20, 2026 - 02:09:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 73 : backward pass time 11.56 +[Apr 20, 2026 - 02:09:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 73 : optimizer step time 0.97 +[Apr 20, 2026 - 02:09:04 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 73-------------------- +[Apr 20, 2026 - 02:09:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0066 iter count: 1.0 +[Apr 20, 2026 - 02:09:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0066 iter count: 1.0 +[Apr 20, 2026 - 02:09:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 02:09:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 02:10:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.854445 38.969608 38.733582 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.047363 67.164062 66.968750 +util 97.500000 100.000000 92.000000 +[Apr 20, 2026 - 02:10:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 74 : avg iteration time 57.03 seconds +[Apr 20, 2026 - 02:10:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 74 : data loading time 0.00 +[Apr 20, 2026 - 02:10:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 74 : avg forward pass time 15.64 +[Apr 20, 2026 - 02:10:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 74 : backward pass time 12.65 +[Apr 20, 2026 - 02:10:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 74 : optimizer step time 1.58 +[Apr 20, 2026 - 02:10:01 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 74-------------------- +[Apr 20, 2026 - 02:10:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0068 iter count: 1.0 +[Apr 20, 2026 - 02:10:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0068 iter count: 1.0 +[Apr 20, 2026 - 02:10:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 02:10:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 02:12:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.854201 38.969608 38.733829 +peak_gpu_mem_gb 129.238753 129.240298 129.238238 +peak_gpu_mem_reserved_gb 133.157715 133.210938 133.076172 +util 67.750000 74.000000 63.000000 +[Apr 20, 2026 - 02:12:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 75 : avg iteration time 176.25 seconds +[Apr 20, 2026 - 02:12:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 75 : data loading time 0.00 +[Apr 20, 2026 - 02:12:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 75 : avg forward pass time 79.13 +[Apr 20, 2026 - 02:12:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 75 : backward pass time 8.35 +[Apr 20, 2026 - 02:12:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 75 : optimizer step time 0.33 +[Apr 20, 2026 - 02:12:58 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 75-------------------- +[Apr 20, 2026 - 02:12:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 14551.2878 iter count: 1.0 +[Apr 20, 2026 - 02:12:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6608 iter count: 1.0 +[Apr 20, 2026 - 02:12:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 14551.2878iter count: 1.0 +[Apr 20, 2026 - 02:12:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 02:12:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.8595iter count: 1.0 +[Apr 20, 2026 - 02:12:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.0471 iter count: 1.0 +[Apr 20, 2026 - 02:12:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.1686 iter count: 1.0 +[Apr 20, 2026 - 02:12:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 23799.0430iter count: 1.0 +[Apr 20, 2026 - 02:12:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 8.1195 iter count: 1.0 +[Apr 20, 2026 - 02:12:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 228034.3906iter count: 1.0 +[Apr 20, 2026 - 02:12:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6608iter count: 1.0 +[Apr 20, 2026 - 02:12:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.5080 iter count: 1.0 +[Apr 20, 2026 - 02:12:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 1114.3064iter count: 1.0 +[Apr 20, 2026 - 02:12:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.6948 iter count: 1.0 +[Apr 20, 2026 - 02:12:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 1619.1954iter count: 1.0 +[Apr 20, 2026 - 02:12:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 1.7906 iter count: 1.0 +[Apr 20, 2026 - 02:12:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 35.9148 iter count: 1.0 +[Apr 20, 2026 - 02:12:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.0899 iter count: 1.0 +[Apr 20, 2026 - 02:12:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 3568.3181iter count: 1.0 +[Apr 20, 2026 - 02:12:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0059 iter count: 1.0 +[Apr 20, 2026 - 02:12:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 02:13:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.854247 38.969608 38.733829 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.252441 67.380859 66.949219 +util 98.500000 100.000000 96.000000 +[Apr 20, 2026 - 02:13:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 76 : avg iteration time 54.91 seconds +[Apr 20, 2026 - 02:13:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 76 : data loading time 0.00 +[Apr 20, 2026 - 02:13:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 76 : avg forward pass time 14.81 +[Apr 20, 2026 - 02:13:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 76 : backward pass time 11.60 +[Apr 20, 2026 - 02:13:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 76 : optimizer step time 0.98 +[Apr 20, 2026 - 02:13:53 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 76-------------------- +[Apr 20, 2026 - 02:13:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0064 iter count: 1.0 +[Apr 20, 2026 - 02:13:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0064 iter count: 1.0 +[Apr 20, 2026 - 02:13:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 02:13:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 02:14:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.854630 38.969654 38.733807 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.193848 67.339844 66.949219 +util 97.000000 99.000000 92.000000 +[Apr 20, 2026 - 02:14:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 77 : avg iteration time 59.83 seconds +[Apr 20, 2026 - 02:14:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 77 : data loading time 0.00 +[Apr 20, 2026 - 02:14:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 77 : avg forward pass time 16.90 +[Apr 20, 2026 - 02:14:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 77 : backward pass time 12.95 +[Apr 20, 2026 - 02:14:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 77 : optimizer step time 1.39 +[Apr 20, 2026 - 02:14:52 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 77-------------------- +[Apr 20, 2026 - 02:14:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0077 iter count: 1.0 +[Apr 20, 2026 - 02:14:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0077 iter count: 1.0 +[Apr 20, 2026 - 02:14:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 02:14:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 02:15:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.854618 38.969608 38.733784 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.193848 67.359375 67.027344 +util 97.500000 100.000000 94.000000 +[Apr 20, 2026 - 02:15:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 78 : avg iteration time 55.68 seconds +[Apr 20, 2026 - 02:15:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 78 : data loading time 0.00 +[Apr 20, 2026 - 02:15:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 78 : avg forward pass time 15.79 +[Apr 20, 2026 - 02:15:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 78 : backward pass time 11.53 +[Apr 20, 2026 - 02:15:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 78 : optimizer step time 0.99 +[Apr 20, 2026 - 02:15:48 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 78-------------------- +[Apr 20, 2026 - 02:15:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0077 iter count: 1.0 +[Apr 20, 2026 - 02:15:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0077 iter count: 1.0 +[Apr 20, 2026 - 02:15:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 02:15:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 02:16:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.854615 38.969631 38.733833 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 68.702637 70.328125 67.007812 +util 97.000000 100.000000 92.000000 +[Apr 20, 2026 - 02:16:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 79 : avg iteration time 53.52 seconds +[Apr 20, 2026 - 02:16:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 79 : data loading time 0.00 +[Apr 20, 2026 - 02:16:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 79 : avg forward pass time 13.96 +[Apr 20, 2026 - 02:16:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 79 : backward pass time 11.55 +[Apr 20, 2026 - 02:16:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 79 : optimizer step time 0.97 +[Apr 20, 2026 - 02:16:42 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 79-------------------- +[Apr 20, 2026 - 02:16:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0064 iter count: 1.0 +[Apr 20, 2026 - 02:16:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0064 iter count: 1.0 +[Apr 20, 2026 - 02:16:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 02:16:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 02:19:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.855320 38.970592 38.734619 +peak_gpu_mem_gb 129.238753 129.240298 129.238238 +peak_gpu_mem_reserved_gb 133.138184 133.191406 133.076172 +util 71.250000 77.000000 63.000000 +[Apr 20, 2026 - 02:19:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 80 : avg iteration time 179.61 seconds +[Apr 20, 2026 - 02:19:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 80 : data loading time 0.00 +[Apr 20, 2026 - 02:19:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 80 : avg forward pass time 80.12 +[Apr 20, 2026 - 02:19:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 80 : backward pass time 8.44 +[Apr 20, 2026 - 02:19:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 80 : optimizer step time 0.34 +[Apr 20, 2026 - 02:19:41 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 80-------------------- +[Apr 20, 2026 - 02:19:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 78027.4688 iter count: 1.0 +[Apr 20, 2026 - 02:19:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6755 iter count: 1.0 +[Apr 20, 2026 - 02:19:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 78027.4688iter count: 1.0 +[Apr 20, 2026 - 02:19:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 02:19:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.6530iter count: 1.0 +[Apr 20, 2026 - 02:19:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.3482 iter count: 1.0 +[Apr 20, 2026 - 02:19:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.1601 iter count: 1.0 +[Apr 20, 2026 - 02:19:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 108124.6328iter count: 1.0 +[Apr 20, 2026 - 02:19:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 2.0064 iter count: 1.0 +[Apr 20, 2026 - 02:19:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 1656362.0000iter count: 1.0 +[Apr 20, 2026 - 02:19:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6755iter count: 1.0 +[Apr 20, 2026 - 02:19:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 0.3482 iter count: 1.0 +[Apr 20, 2026 - 02:19:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 2.0064 iter count: 1.0 +[Apr 20, 2026 - 02:19:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 6.1363 iter count: 1.0 +[Apr 20, 2026 - 02:19:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 213740.5469iter count: 1.0 +[Apr 20, 2026 - 02:19:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 5.4142 iter count: 1.0 +[Apr 20, 2026 - 02:19:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 50430.7188iter count: 1.0 +[Apr 20, 2026 - 02:19:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 2.8694 iter count: 1.0 +[Apr 20, 2026 - 02:19:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 310.7204iter count: 1.0 +[Apr 20, 2026 - 02:19:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0061 iter count: 1.0 +[Apr 20, 2026 - 02:19:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 02:20:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.855314 38.970612 38.734596 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.354980 67.419922 67.320312 +util 97.500000 100.000000 92.000000 +[Apr 20, 2026 - 02:20:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 81 : avg iteration time 54.33 seconds +[Apr 20, 2026 - 02:20:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 81 : data loading time 0.00 +[Apr 20, 2026 - 02:20:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 81 : avg forward pass time 14.75 +[Apr 20, 2026 - 02:20:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 81 : backward pass time 11.57 +[Apr 20, 2026 - 02:20:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 81 : optimizer step time 0.98 +[Apr 20, 2026 - 02:20:35 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 81-------------------- +[Apr 20, 2026 - 02:20:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0067 iter count: 1.0 +[Apr 20, 2026 - 02:20:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0067 iter count: 1.0 +[Apr 20, 2026 - 02:20:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 02:20:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 02:21:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.855311 38.970592 38.734596 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.306152 67.437500 66.970703 +util 98.250000 100.000000 94.000000 +[Apr 20, 2026 - 02:21:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 82 : avg iteration time 60.02 seconds +[Apr 20, 2026 - 02:21:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 82 : data loading time 0.00 +[Apr 20, 2026 - 02:21:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 82 : avg forward pass time 17.06 +[Apr 20, 2026 - 02:21:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 82 : backward pass time 12.79 +[Apr 20, 2026 - 02:21:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 82 : optimizer step time 1.55 +[Apr 20, 2026 - 02:21:36 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 82-------------------- +[Apr 20, 2026 - 02:21:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0066 iter count: 1.0 +[Apr 20, 2026 - 02:21:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0066 iter count: 1.0 +[Apr 20, 2026 - 02:21:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 02:21:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 02:22:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.855314 38.970592 38.734619 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.120605 67.144531 67.105469 +util 98.500000 100.000000 96.000000 +[Apr 20, 2026 - 02:22:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 83 : avg iteration time 55.61 seconds +[Apr 20, 2026 - 02:22:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 83 : data loading time 0.00 +[Apr 20, 2026 - 02:22:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 83 : avg forward pass time 15.76 +[Apr 20, 2026 - 02:22:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 83 : backward pass time 11.56 +[Apr 20, 2026 - 02:22:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 83 : optimizer step time 0.98 +[Apr 20, 2026 - 02:22:31 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 83-------------------- +[Apr 20, 2026 - 02:22:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0096 iter count: 1.0 +[Apr 20, 2026 - 02:22:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[Apr 20, 2026 - 02:22:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 02:22:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 02:23:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.855362 38.970570 38.734604 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.296387 67.437500 66.970703 +util 95.750000 100.000000 91.000000 +[Apr 20, 2026 - 02:23:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 84 : avg iteration time 53.65 seconds +[Apr 20, 2026 - 02:23:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 84 : data loading time 0.00 +[Apr 20, 2026 - 02:23:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 84 : avg forward pass time 14.08 +[Apr 20, 2026 - 02:23:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 84 : backward pass time 11.58 +[Apr 20, 2026 - 02:23:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 84 : optimizer step time 0.99 +[Apr 20, 2026 - 02:23:25 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 84-------------------- +[Apr 20, 2026 - 02:23:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0064 iter count: 1.0 +[Apr 20, 2026 - 02:23:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0064 iter count: 1.0 +[Apr 20, 2026 - 02:23:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 02:23:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 02:26:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.855111 38.970695 38.734764 +peak_gpu_mem_gb 129.238753 129.240298 129.238238 +peak_gpu_mem_reserved_gb 133.133301 133.152344 133.093750 +util 69.750000 76.000000 66.000000 +[Apr 20, 2026 - 02:26:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 85 : avg iteration time 186.66 seconds +[Apr 20, 2026 - 02:26:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 85 : data loading time 0.00 +[Apr 20, 2026 - 02:26:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 85 : avg forward pass time 84.25 +[Apr 20, 2026 - 02:26:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 85 : backward pass time 8.41 +[Apr 20, 2026 - 02:26:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 85 : optimizer step time 0.34 +[Apr 20, 2026 - 02:26:31 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 85-------------------- +[Apr 20, 2026 - 02:26:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 18041.7122 iter count: 1.0 +[Apr 20, 2026 - 02:26:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6638 iter count: 1.0 +[Apr 20, 2026 - 02:26:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 18041.7122iter count: 1.0 +[Apr 20, 2026 - 02:26:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 02:26:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.9806iter count: 1.0 +[Apr 20, 2026 - 02:26:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.2888 iter count: 1.0 +[Apr 20, 2026 - 02:26:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.4898 iter count: 1.0 +[Apr 20, 2026 - 02:26:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 33356.5391iter count: 1.0 +[Apr 20, 2026 - 02:26:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 13.1663iter count: 1.0 +[Apr 20, 2026 - 02:26:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 433456.8125iter count: 1.0 +[Apr 20, 2026 - 02:26:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6638iter count: 1.0 +[Apr 20, 2026 - 02:26:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.9649 iter count: 1.0 +[Apr 20, 2026 - 02:26:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 2778.6006iter count: 1.0 +[Apr 20, 2026 - 02:26:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.2468 iter count: 1.0 +[Apr 20, 2026 - 02:26:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 660.9330iter count: 1.0 +[Apr 20, 2026 - 02:26:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 2.4147 iter count: 1.0 +[Apr 20, 2026 - 02:26:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 125.1353iter count: 1.0 +[Apr 20, 2026 - 02:26:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.6715 iter count: 1.0 +[Apr 20, 2026 - 02:26:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 11417.6260iter count: 1.0 +[Apr 20, 2026 - 02:26:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0065 iter count: 1.0 +[Apr 20, 2026 - 02:26:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 02:27:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.855170 38.970695 38.734764 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 68.678223 70.171875 66.951172 +util 98.500000 100.000000 95.000000 +[Apr 20, 2026 - 02:27:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 86 : avg iteration time 54.70 seconds +[Apr 20, 2026 - 02:27:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 86 : data loading time 0.00 +[Apr 20, 2026 - 02:27:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 86 : avg forward pass time 14.65 +[Apr 20, 2026 - 02:27:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 86 : backward pass time 11.53 +[Apr 20, 2026 - 02:27:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 86 : optimizer step time 1.05 +[Apr 20, 2026 - 02:27:26 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 86-------------------- +[Apr 20, 2026 - 02:27:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0073 iter count: 1.0 +[Apr 20, 2026 - 02:27:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0073 iter count: 1.0 +[Apr 20, 2026 - 02:27:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 02:27:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 02:28:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.855480 38.970695 38.734764 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.403809 67.458984 67.359375 +util 97.250000 99.000000 94.000000 +[Apr 20, 2026 - 02:28:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 87 : avg iteration time 50.69 seconds +[Apr 20, 2026 - 02:28:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 87 : data loading time 0.00 +[Apr 20, 2026 - 02:28:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 87 : avg forward pass time 12.59 +[Apr 20, 2026 - 02:28:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 87 : backward pass time 12.88 +[Apr 20, 2026 - 02:28:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 87 : optimizer step time 1.07 +[Apr 20, 2026 - 02:28:17 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 87-------------------- +[Apr 20, 2026 - 02:28:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0064 iter count: 1.0 +[Apr 20, 2026 - 02:28:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0064 iter count: 1.0 +[Apr 20, 2026 - 02:28:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 02:28:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 02:29:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.855739 38.970703 38.734787 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.262207 67.378906 66.970703 +util 97.500000 99.000000 95.000000 +[Apr 20, 2026 - 02:29:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 88 : avg iteration time 55.41 seconds +[Apr 20, 2026 - 02:29:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 88 : data loading time 0.00 +[Apr 20, 2026 - 02:29:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 88 : avg forward pass time 15.61 +[Apr 20, 2026 - 02:29:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 88 : backward pass time 11.58 +[Apr 20, 2026 - 02:29:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 88 : optimizer step time 0.99 +[Apr 20, 2026 - 02:29:12 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 88-------------------- +[Apr 20, 2026 - 02:29:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0091 iter count: 1.0 +[Apr 20, 2026 - 02:29:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[Apr 20, 2026 - 02:29:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 02:29:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 02:30:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.855861 38.970695 38.734764 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.384277 67.439453 67.359375 +util 98.000000 100.000000 94.000000 +[Apr 20, 2026 - 02:30:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 89 : avg iteration time 50.86 seconds +[Apr 20, 2026 - 02:30:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 89 : data loading time 0.00 +[Apr 20, 2026 - 02:30:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 89 : avg forward pass time 12.61 +[Apr 20, 2026 - 02:30:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 89 : backward pass time 11.59 +[Apr 20, 2026 - 02:30:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 89 : optimizer step time 0.98 +[Apr 20, 2026 - 02:30:03 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 89-------------------- +[Apr 20, 2026 - 02:30:03 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0066 iter count: 1.0 +[Apr 20, 2026 - 02:30:03 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0066 iter count: 1.0 +[Apr 20, 2026 - 02:30:03 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 02:30:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 02:32:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.855288 38.970695 38.734764 +peak_gpu_mem_gb 129.238753 129.240298 129.238238 +peak_gpu_mem_reserved_gb 133.104492 133.156250 133.074219 +util 71.000000 77.000000 63.000000 +[Apr 20, 2026 - 02:32:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 90 : avg iteration time 175.85 seconds +[Apr 20, 2026 - 02:32:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 90 : data loading time 0.00 +[Apr 20, 2026 - 02:32:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 90 : avg forward pass time 78.89 +[Apr 20, 2026 - 02:32:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 90 : backward pass time 8.44 +[Apr 20, 2026 - 02:32:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 90 : optimizer step time 0.33 +[Apr 20, 2026 - 02:32:59 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 90-------------------- +[Apr 20, 2026 - 02:32:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 65318.7231 iter count: 1.0 +[Apr 20, 2026 - 02:32:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6846 iter count: 1.0 +[Apr 20, 2026 - 02:32:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 65318.7231iter count: 1.0 +[Apr 20, 2026 - 02:32:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 02:32:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.8904iter count: 1.0 +[Apr 20, 2026 - 02:32:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.9943 iter count: 1.0 +[Apr 20, 2026 - 02:32:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.0259 iter count: 1.0 +[Apr 20, 2026 - 02:32:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 99192.5938iter count: 1.0 +[Apr 20, 2026 - 02:32:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 7.3046 iter count: 1.0 +[Apr 20, 2026 - 02:32:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 1266456.0000iter count: 1.0 +[Apr 20, 2026 - 02:32:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6846iter count: 1.0 +[Apr 20, 2026 - 02:32:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 2.6470 iter count: 1.0 +[Apr 20, 2026 - 02:32:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 199.1256iter count: 1.0 +[Apr 20, 2026 - 02:32:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 5.1226 iter count: 1.0 +[Apr 20, 2026 - 02:32:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 28144.7793iter count: 1.0 +[Apr 20, 2026 - 02:32:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 3.1503 iter count: 1.0 +[Apr 20, 2026 - 02:32:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 544.9487iter count: 1.0 +[Apr 20, 2026 - 02:32:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 5.8160 iter count: 1.0 +[Apr 20, 2026 - 02:32:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 112641.3047iter count: 1.0 +[Apr 20, 2026 - 02:32:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0066 iter count: 1.0 +[Apr 20, 2026 - 02:32:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 02:34:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.855412 38.970711 38.734764 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.232910 67.380859 66.949219 +util 95.750000 99.000000 92.000000 +[Apr 20, 2026 - 02:34:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 91 : avg iteration time 62.46 seconds +[Apr 20, 2026 - 02:34:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 91 : data loading time 0.00 +[Apr 20, 2026 - 02:34:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 91 : avg forward pass time 18.82 +[Apr 20, 2026 - 02:34:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 91 : backward pass time 11.54 +[Apr 20, 2026 - 02:34:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 91 : optimizer step time 1.03 +[Apr 20, 2026 - 02:34:01 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 91-------------------- +[Apr 20, 2026 - 02:34:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0096 iter count: 1.0 +[Apr 20, 2026 - 02:34:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[Apr 20, 2026 - 02:34:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 02:34:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 02:34:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.855721 38.970695 38.734764 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.096191 67.164062 66.988281 +util 98.500000 100.000000 96.000000 +[Apr 20, 2026 - 02:34:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 92 : avg iteration time 54.11 seconds +[Apr 20, 2026 - 02:34:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 92 : data loading time 0.00 +[Apr 20, 2026 - 02:34:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 92 : avg forward pass time 14.25 +[Apr 20, 2026 - 02:34:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 92 : backward pass time 12.97 +[Apr 20, 2026 - 02:34:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 92 : optimizer step time 1.12 +[Apr 20, 2026 - 02:34:56 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 92-------------------- +[Apr 20, 2026 - 02:34:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0061 iter count: 1.0 +[Apr 20, 2026 - 02:34:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0061 iter count: 1.0 +[Apr 20, 2026 - 02:34:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 02:34:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 02:35:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.855733 38.970737 38.734764 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.203613 67.398438 66.988281 +util 98.250000 100.000000 96.000000 +[Apr 20, 2026 - 02:35:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 93 : avg iteration time 49.92 seconds +[Apr 20, 2026 - 02:35:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 93 : data loading time 0.00 +[Apr 20, 2026 - 02:35:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 93 : avg forward pass time 12.91 +[Apr 20, 2026 - 02:35:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 93 : backward pass time 11.55 +[Apr 20, 2026 - 02:35:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 93 : optimizer step time 0.98 +[Apr 20, 2026 - 02:35:45 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 93-------------------- +[Apr 20, 2026 - 02:35:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0067 iter count: 1.0 +[Apr 20, 2026 - 02:35:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0067 iter count: 1.0 +[Apr 20, 2026 - 02:35:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 02:35:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 02:36:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.855618 38.970699 38.734787 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.203613 67.417969 66.970703 +util 97.000000 100.000000 91.000000 +[Apr 20, 2026 - 02:36:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 94 : avg iteration time 58.83 seconds +[Apr 20, 2026 - 02:36:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 94 : data loading time 0.00 +[Apr 20, 2026 - 02:36:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 94 : avg forward pass time 16.84 +[Apr 20, 2026 - 02:36:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 94 : backward pass time 11.55 +[Apr 20, 2026 - 02:36:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 94 : optimizer step time 1.01 +[Apr 20, 2026 - 02:36:44 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 94-------------------- +[Apr 20, 2026 - 02:36:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0073 iter count: 1.0 +[Apr 20, 2026 - 02:36:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0073 iter count: 1.0 +[Apr 20, 2026 - 02:36:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 02:36:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 02:39:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.855474 38.970764 38.734776 +peak_gpu_mem_gb 129.238753 129.240298 129.238238 +peak_gpu_mem_reserved_gb 133.128418 133.134766 133.113281 +util 70.250000 77.000000 66.000000 +[Apr 20, 2026 - 02:39:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 95 : avg iteration time 174.57 seconds +[Apr 20, 2026 - 02:39:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 95 : data loading time 0.00 +[Apr 20, 2026 - 02:39:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 95 : avg forward pass time 77.84 +[Apr 20, 2026 - 02:39:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 95 : backward pass time 8.38 +[Apr 20, 2026 - 02:39:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 95 : optimizer step time 0.33 +[Apr 20, 2026 - 02:39:39 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 95-------------------- +[Apr 20, 2026 - 02:39:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 120029.2611 iter count: 1.0 +[Apr 20, 2026 - 02:39:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6070 iter count: 1.0 +[Apr 20, 2026 - 02:39:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 120029.2611iter count: 1.0 +[Apr 20, 2026 - 02:39:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 02:39:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.5141iter count: 1.0 +[Apr 20, 2026 - 02:39:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.5956 iter count: 1.0 +[Apr 20, 2026 - 02:39:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.6211 iter count: 1.0 +[Apr 20, 2026 - 02:39:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 144913.9062iter count: 1.0 +[Apr 20, 2026 - 02:39:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 3.2913 iter count: 1.0 +[Apr 20, 2026 - 02:39:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 4164649.5000iter count: 1.0 +[Apr 20, 2026 - 02:39:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6070iter count: 1.0 +[Apr 20, 2026 - 02:39:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 0.5956 iter count: 1.0 +[Apr 20, 2026 - 02:39:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 3.2913 iter count: 1.0 +[Apr 20, 2026 - 02:39:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.2011 iter count: 1.0 +[Apr 20, 2026 - 02:39:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 603.1468iter count: 1.0 +[Apr 20, 2026 - 02:39:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 3.0273 iter count: 1.0 +[Apr 20, 2026 - 02:39:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 426.0269iter count: 1.0 +[Apr 20, 2026 - 02:39:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 5.4096 iter count: 1.0 +[Apr 20, 2026 - 02:39:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 49969.0703iter count: 1.0 +[Apr 20, 2026 - 02:39:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0063 iter count: 1.0 +[Apr 20, 2026 - 02:39:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 02:40:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.855659 38.970764 38.734776 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.403809 67.419922 67.359375 +util 98.500000 100.000000 96.000000 +[Apr 20, 2026 - 02:40:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 96 : avg iteration time 59.09 seconds +[Apr 20, 2026 - 02:40:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 96 : data loading time 0.00 +[Apr 20, 2026 - 02:40:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 96 : avg forward pass time 17.46 +[Apr 20, 2026 - 02:40:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 96 : backward pass time 11.59 +[Apr 20, 2026 - 02:40:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 96 : optimizer step time 0.99 +[Apr 20, 2026 - 02:40:38 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 96-------------------- +[Apr 20, 2026 - 02:40:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0069 iter count: 1.0 +[Apr 20, 2026 - 02:40:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0069 iter count: 1.0 +[Apr 20, 2026 - 02:40:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 02:40:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 02:41:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.855725 38.970764 38.734783 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 68.053223 70.230469 67.029297 +util 98.750000 100.000000 96.000000 +[Apr 20, 2026 - 02:41:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 97 : avg iteration time 50.60 seconds +[Apr 20, 2026 - 02:41:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 97 : data loading time 0.00 +[Apr 20, 2026 - 02:41:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 97 : avg forward pass time 12.89 +[Apr 20, 2026 - 02:41:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 97 : backward pass time 11.63 +[Apr 20, 2026 - 02:41:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 97 : optimizer step time 1.40 +[Apr 20, 2026 - 02:41:29 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 97-------------------- +[Apr 20, 2026 - 02:41:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0068 iter count: 1.0 +[Apr 20, 2026 - 02:41:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0068 iter count: 1.0 +[Apr 20, 2026 - 02:41:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 02:41:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 02:42:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.85572 38.970764 38.734776 +peak_gpu_mem_gb 65.49830 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.26709 67.457031 67.009766 +util 97.50000 99.000000 93.000000 +[Apr 20, 2026 - 02:42:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 98 : avg iteration time 50.57 seconds +[Apr 20, 2026 - 02:42:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 98 : data loading time 0.00 +[Apr 20, 2026 - 02:42:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 98 : avg forward pass time 12.76 +[Apr 20, 2026 - 02:42:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 98 : backward pass time 12.45 +[Apr 20, 2026 - 02:42:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 98 : optimizer step time 1.02 +[Apr 20, 2026 - 02:42:19 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 98-------------------- +[Apr 20, 2026 - 02:42:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0075 iter count: 1.0 +[Apr 20, 2026 - 02:42:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0075 iter count: 1.0 +[Apr 20, 2026 - 02:42:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 02:42:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 02:43:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.855723 38.970764 38.734776 +peak_gpu_mem_gb 65.498300 65.499845 65.497785 +peak_gpu_mem_reserved_gb 67.315918 67.476562 67.009766 +util 98.000000 100.000000 93.000000 +[Apr 20, 2026 - 02:43:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 99 : avg iteration time 52.47 seconds +[Apr 20, 2026 - 02:43:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 99 : data loading time 0.00 +[Apr 20, 2026 - 02:43:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 99 : avg forward pass time 14.04 +[Apr 20, 2026 - 02:43:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 99 : backward pass time 11.60 +[Apr 20, 2026 - 02:43:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 99 : optimizer step time 0.98 +[Apr 20, 2026 - 02:43:12 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 99-------------------- +[Apr 20, 2026 - 02:43:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0072 iter count: 1.0 +[Apr 20, 2026 - 02:43:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0072 iter count: 1.0 +[Apr 20, 2026 - 02:43:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 02:43:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 02:46:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 38.855736 38.970783 38.734791 +peak_gpu_mem_gb 129.238753 129.240298 129.238238 +peak_gpu_mem_reserved_gb 133.128418 133.152344 133.093750 +util 73.500000 80.000000 65.000000 +[Apr 20, 2026 - 02:46:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 100 : avg iteration time 183.41 seconds +[Apr 20, 2026 - 02:46:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 100 : data loading time 0.00 +[Apr 20, 2026 - 02:46:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 100 : avg forward pass time 82.23 +[Apr 20, 2026 - 02:46:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 100 : backward pass time 9.75 +[Apr 20, 2026 - 02:46:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 100 : optimizer step time 0.33 +[Apr 20, 2026 - 02:46:15 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 100-------------------- +[Apr 20, 2026 - 02:46:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 18328.4342 iter count: 1.0 +[Apr 20, 2026 - 02:46:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6257 iter count: 1.0 +[Apr 20, 2026 - 02:46:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 18328.4342iter count: 1.0 +[Apr 20, 2026 - 02:46:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 02:46:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.9155iter count: 1.0 +[Apr 20, 2026 - 02:46:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.8857 iter count: 1.0 +[Apr 20, 2026 - 02:46:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.2104 iter count: 1.0 +[Apr 20, 2026 - 02:46:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 22682.7188iter count: 1.0 +[Apr 20, 2026 - 02:46:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 5.8787 iter count: 1.0 +[Apr 20, 2026 - 02:46:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 247897.9844iter count: 1.0 +[Apr 20, 2026 - 02:46:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6257iter count: 1.0 +[Apr 20, 2026 - 02:46:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.0615 iter count: 1.0 +[Apr 20, 2026 - 02:46:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 3371.3027iter count: 1.0 +[Apr 20, 2026 - 02:46:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.6659 iter count: 1.0 +[Apr 20, 2026 - 02:46:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 11290.9482iter count: 1.0 +[Apr 20, 2026 - 02:46:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 5.1064 iter count: 1.0 +[Apr 20, 2026 - 02:46:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 27247.5000iter count: 1.0 +[Apr 20, 2026 - 02:46:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.8465 iter count: 1.0 +[Apr 20, 2026 - 02:46:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 16204.0117iter count: 1.0 +[Apr 20, 2026 - 02:46:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0074 iter count: 1.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[Apr 20, 2026 - 02:46:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 23.67 seconds +[Apr 20, 2026 - 02:46:39 | INFO | fastgen.trainer:save_checkpoint:266 ] Saving checkpoint iteration 100 +[Apr 20, 2026 - 02:46:39 | INFO | fastgen.utils.checkpointer:save:301 ] Saving FSDP model to prefix /tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000100 +[Apr 20, 2026 - 02:46:55 | SUCCESS | fastgen.utils.checkpointer:save:329 ] Model saved at iteration 100 +[Apr 20, 2026 - 02:46:55 | INFO | fastgen.trainer:validate:358 ] Validation iteration 100 +[Apr 20, 2026 - 02:46:56 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 02:46:59 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 3.3s +[Apr 20, 2026 - 02:47:03 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 02:47:06 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 3.0s +[Apr 20, 2026 - 02:47:09 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 02:47:12 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 2.9s +[Apr 20, 2026 - 02:47:15 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 02:47:18 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 2.9s +[Apr 20, 2026 - 02:47:22 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 02:47:25 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 2.9s +[Apr 20, 2026 - 02:47:28 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 02:47:31 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 2.9s +[Apr 20, 2026 - 02:47:34 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 02:47:37 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 2.9s +[Apr 20, 2026 - 02:47:40 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 02:47:43 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 2.9s +[Apr 20, 2026 - 02:47:46 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 02:47:49 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 2.9s +[Apr 20, 2026 - 02:47:53 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 02:47:56 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 2.9s +[Apr 20, 2026 - 02:47:59 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging val0 stats at iteration 100-------------------- +[Apr 20, 2026 - 02:47:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0000 iter count: 10.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[Apr 20, 2026 - 02:48:16 | INFO | fastgen.callbacks.wandb:on_validation_end:683 ] Logged 10 val videos at iteration 100 +[Apr 20, 2026 - 02:49:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121354 43.233120 42.990475 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.830566 69.937500 69.529297 +util 99.000000 100.000000 97.000000 +[Apr 20, 2026 - 02:49:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 101 : avg iteration time 184.90 seconds +[Apr 20, 2026 - 02:49:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 101 : data loading time 0.00 +[Apr 20, 2026 - 02:49:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 101 : avg forward pass time 18.89 +[Apr 20, 2026 - 02:49:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 101 : backward pass time 11.77 +[Apr 20, 2026 - 02:49:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 101 : optimizer step time 1.49 +[Apr 20, 2026 - 02:49:20 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 101-------------------- +[Apr 20, 2026 - 02:49:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0083 iter count: 1.0 +[Apr 20, 2026 - 02:49:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[Apr 20, 2026 - 02:49:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 02:49:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 02:50:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121366 43.233120 42.990498 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.864746 69.957031 69.587891 +util 96.000000 100.000000 93.000000 +[Apr 20, 2026 - 02:50:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 102 : avg iteration time 55.55 seconds +[Apr 20, 2026 - 02:50:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 102 : data loading time 0.00 +[Apr 20, 2026 - 02:50:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 102 : avg forward pass time 15.71 +[Apr 20, 2026 - 02:50:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 102 : backward pass time 11.57 +[Apr 20, 2026 - 02:50:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 102 : optimizer step time 0.99 +[Apr 20, 2026 - 02:50:15 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 102-------------------- +[Apr 20, 2026 - 02:50:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0074 iter count: 1.0 +[Apr 20, 2026 - 02:50:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0074 iter count: 1.0 +[Apr 20, 2026 - 02:50:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 02:50:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 02:51:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121495 43.233513 42.990623 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.771973 69.917969 69.529297 +util 98.000000 100.000000 95.000000 +[Apr 20, 2026 - 02:51:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 103 : avg iteration time 53.23 seconds +[Apr 20, 2026 - 02:51:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 103 : data loading time 0.00 +[Apr 20, 2026 - 02:51:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 103 : avg forward pass time 14.15 +[Apr 20, 2026 - 02:51:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 103 : backward pass time 11.56 +[Apr 20, 2026 - 02:51:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 103 : optimizer step time 0.98 +[Apr 20, 2026 - 02:51:09 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 103-------------------- +[Apr 20, 2026 - 02:51:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0076 iter count: 1.0 +[Apr 20, 2026 - 02:51:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0076 iter count: 1.0 +[Apr 20, 2026 - 02:51:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 02:51:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 02:52:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121537 43.233536 42.990646 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.786621 69.957031 69.722656 +util 98.000000 100.000000 94.000000 +[Apr 20, 2026 - 02:52:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 104 : avg iteration time 64.26 seconds +[Apr 20, 2026 - 02:52:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 104 : data loading time 0.00 +[Apr 20, 2026 - 02:52:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 104 : avg forward pass time 20.10 +[Apr 20, 2026 - 02:52:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 104 : backward pass time 11.53 +[Apr 20, 2026 - 02:52:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 104 : optimizer step time 0.98 +[Apr 20, 2026 - 02:52:13 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 104-------------------- +[Apr 20, 2026 - 02:52:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0075 iter count: 1.0 +[Apr 20, 2026 - 02:52:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0075 iter count: 1.0 +[Apr 20, 2026 - 02:52:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 02:52:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 02:55:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121184 43.230198 42.991405 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.479004 135.498047 135.439453 +util 72.000000 78.000000 64.000000 +[Apr 20, 2026 - 02:55:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 105 : avg iteration time 185.86 seconds +[Apr 20, 2026 - 02:55:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 105 : data loading time 0.00 +[Apr 20, 2026 - 02:55:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 105 : avg forward pass time 84.40 +[Apr 20, 2026 - 02:55:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 105 : backward pass time 8.43 +[Apr 20, 2026 - 02:55:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 105 : optimizer step time 0.33 +[Apr 20, 2026 - 02:55:19 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 105-------------------- +[Apr 20, 2026 - 02:55:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 895244.6261 iter count: 1.0 +[Apr 20, 2026 - 02:55:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6753 iter count: 1.0 +[Apr 20, 2026 - 02:55:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 895244.6261iter count: 1.0 +[Apr 20, 2026 - 02:55:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 02:55:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.9858iter count: 1.0 +[Apr 20, 2026 - 02:55:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.7603 iter count: 1.0 +[Apr 20, 2026 - 02:55:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 8.6195 iter count: 1.0 +[Apr 20, 2026 - 02:55:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 1008821.3750iter count: 1.0 +[Apr 20, 2026 - 02:55:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 4.5751 iter count: 1.0 +[Apr 20, 2026 - 02:55:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 30673298.0000iter count: 1.0 +[Apr 20, 2026 - 02:55:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6753iter count: 1.0 +[Apr 20, 2026 - 02:55:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 6.4964 iter count: 1.0 +[Apr 20, 2026 - 02:55:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 439229.7188iter count: 1.0 +[Apr 20, 2026 - 02:55:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.5770 iter count: 1.0 +[Apr 20, 2026 - 02:55:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 9452.0957iter count: 1.0 +[Apr 20, 2026 - 02:55:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 8.6195 iter count: 1.0 +[Apr 20, 2026 - 02:55:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 30673298.0000iter count: 1.0 +[Apr 20, 2026 - 02:55:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 1.2574 iter count: 1.0 +[Apr 20, 2026 - 02:55:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 12.3643 iter count: 1.0 +[Apr 20, 2026 - 02:55:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0075 iter count: 1.0 +[Apr 20, 2026 - 02:55:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 02:56:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121625 43.231922 42.991444 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.426270 72.341797 69.683594 +util 98.500000 100.000000 95.000000 +[Apr 20, 2026 - 02:56:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 106 : avg iteration time 57.76 seconds +[Apr 20, 2026 - 02:56:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 106 : data loading time 0.00 +[Apr 20, 2026 - 02:56:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 106 : avg forward pass time 16.36 +[Apr 20, 2026 - 02:56:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 106 : backward pass time 12.48 +[Apr 20, 2026 - 02:56:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 106 : optimizer step time 1.00 +[Apr 20, 2026 - 02:56:17 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 106-------------------- +[Apr 20, 2026 - 02:56:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0082 iter count: 1.0 +[Apr 20, 2026 - 02:56:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[Apr 20, 2026 - 02:56:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 02:56:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 02:57:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121660 43.232029 42.991409 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 71.178223 72.867188 69.761719 +util 97.000000 99.000000 94.000000 +[Apr 20, 2026 - 02:57:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 107 : avg iteration time 52.11 seconds +[Apr 20, 2026 - 02:57:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 107 : data loading time 0.00 +[Apr 20, 2026 - 02:57:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 107 : avg forward pass time 13.98 +[Apr 20, 2026 - 02:57:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 107 : backward pass time 11.57 +[Apr 20, 2026 - 02:57:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 107 : optimizer step time 0.99 +[Apr 20, 2026 - 02:57:09 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 107-------------------- +[Apr 20, 2026 - 02:57:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0089 iter count: 1.0 +[Apr 20, 2026 - 02:57:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[Apr 20, 2026 - 02:57:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 02:57:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 02:57:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121699 43.232166 42.991428 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 71.241699 72.847656 69.820312 +util 97.750000 100.000000 93.000000 +[Apr 20, 2026 - 02:57:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 108 : avg iteration time 47.66 seconds +[Apr 20, 2026 - 02:57:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 108 : data loading time 0.00 +[Apr 20, 2026 - 02:57:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 108 : avg forward pass time 11.34 +[Apr 20, 2026 - 02:57:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 108 : backward pass time 11.54 +[Apr 20, 2026 - 02:57:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 108 : optimizer step time 1.02 +[Apr 20, 2026 - 02:57:56 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 108-------------------- +[Apr 20, 2026 - 02:57:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0077 iter count: 1.0 +[Apr 20, 2026 - 02:57:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0077 iter count: 1.0 +[Apr 20, 2026 - 02:57:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 02:57:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 02:58:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121739 43.232319 42.991451 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 71.153809 72.789062 69.761719 +util 95.750000 100.000000 92.000000 +[Apr 20, 2026 - 02:58:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 109 : avg iteration time 53.11 seconds +[Apr 20, 2026 - 02:58:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 109 : data loading time 0.00 +[Apr 20, 2026 - 02:58:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 109 : avg forward pass time 14.34 +[Apr 20, 2026 - 02:58:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 109 : backward pass time 11.88 +[Apr 20, 2026 - 02:58:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 109 : optimizer step time 0.98 +[Apr 20, 2026 - 02:58:49 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 109-------------------- +[Apr 20, 2026 - 02:58:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0087 iter count: 1.0 +[Apr 20, 2026 - 02:58:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[Apr 20, 2026 - 02:58:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 02:58:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 03:01:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121520 43.231457 42.991405 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.469238 135.478516 135.441406 +util 71.250000 76.000000 65.000000 +[Apr 20, 2026 - 03:01:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 110 : avg iteration time 183.49 seconds +[Apr 20, 2026 - 03:01:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 110 : data loading time 0.00 +[Apr 20, 2026 - 03:01:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 110 : avg forward pass time 83.18 +[Apr 20, 2026 - 03:01:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 110 : backward pass time 8.42 +[Apr 20, 2026 - 03:01:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 110 : optimizer step time 0.33 +[Apr 20, 2026 - 03:01:53 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 110-------------------- +[Apr 20, 2026 - 03:01:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 25386.2444 iter count: 1.0 +[Apr 20, 2026 - 03:01:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6736 iter count: 1.0 +[Apr 20, 2026 - 03:01:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 25386.2444iter count: 1.0 +[Apr 20, 2026 - 03:01:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 03:01:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.9591iter count: 1.0 +[Apr 20, 2026 - 03:01:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.2381 iter count: 1.0 +[Apr 20, 2026 - 03:01:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.8904 iter count: 1.0 +[Apr 20, 2026 - 03:01:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 64537.3906iter count: 1.0 +[Apr 20, 2026 - 03:01:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 11.8950iter count: 1.0 +[Apr 20, 2026 - 03:01:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 965935.4375iter count: 1.0 +[Apr 20, 2026 - 03:01:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6736iter count: 1.0 +[Apr 20, 2026 - 03:01:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.4993 iter count: 1.0 +[Apr 20, 2026 - 03:01:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 1095.1261iter count: 1.0 +[Apr 20, 2026 - 03:01:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.4093 iter count: 1.0 +[Apr 20, 2026 - 03:01:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 914.7033iter count: 1.0 +[Apr 20, 2026 - 03:01:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 2.4101 iter count: 1.0 +[Apr 20, 2026 - 03:01:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 123.9970iter count: 1.0 +[Apr 20, 2026 - 03:01:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 5.6444 iter count: 1.0 +[Apr 20, 2026 - 03:01:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 79925.6953iter count: 1.0 +[Apr 20, 2026 - 03:01:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0077 iter count: 1.0 +[Apr 20, 2026 - 03:01:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 03:02:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121545 43.231514 42.991432 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.465332 72.691406 69.722656 +util 98.500000 100.000000 95.000000 +[Apr 20, 2026 - 03:02:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 111 : avg iteration time 58.08 seconds +[Apr 20, 2026 - 03:02:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 111 : data loading time 0.00 +[Apr 20, 2026 - 03:02:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 111 : avg forward pass time 15.96 +[Apr 20, 2026 - 03:02:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 111 : backward pass time 11.79 +[Apr 20, 2026 - 03:02:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 111 : optimizer step time 1.50 +[Apr 20, 2026 - 03:02:51 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 111-------------------- +[Apr 20, 2026 - 03:02:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0092 iter count: 1.0 +[Apr 20, 2026 - 03:02:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[Apr 20, 2026 - 03:02:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 03:02:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 03:03:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121541 43.231533 42.991425 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.859863 69.900391 69.820312 +util 97.750000 100.000000 93.000000 +[Apr 20, 2026 - 03:03:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 112 : avg iteration time 58.18 seconds +[Apr 20, 2026 - 03:03:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 112 : data loading time 0.00 +[Apr 20, 2026 - 03:03:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 112 : avg forward pass time 17.03 +[Apr 20, 2026 - 03:03:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 112 : backward pass time 11.57 +[Apr 20, 2026 - 03:03:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 112 : optimizer step time 0.99 +[Apr 20, 2026 - 03:03:49 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 112-------------------- +[Apr 20, 2026 - 03:03:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0082 iter count: 1.0 +[Apr 20, 2026 - 03:03:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[Apr 20, 2026 - 03:03:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 03:03:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 03:04:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121550 43.231518 42.991428 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.533691 72.789062 69.742188 +util 96.750000 100.000000 94.000000 +[Apr 20, 2026 - 03:04:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 113 : avg iteration time 56.06 seconds +[Apr 20, 2026 - 03:04:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 113 : data loading time 0.00 +[Apr 20, 2026 - 03:04:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 113 : avg forward pass time 15.29 +[Apr 20, 2026 - 03:04:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 113 : backward pass time 11.52 +[Apr 20, 2026 - 03:04:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 113 : optimizer step time 1.00 +[Apr 20, 2026 - 03:04:45 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 113-------------------- +[Apr 20, 2026 - 03:04:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0070 iter count: 1.0 +[Apr 20, 2026 - 03:04:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0070 iter count: 1.0 +[Apr 20, 2026 - 03:04:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 03:04:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 03:05:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121598 43.231720 42.991428 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.621582 72.847656 69.841797 +util 97.250000 100.000000 92.000000 +[Apr 20, 2026 - 03:05:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 114 : avg iteration time 52.72 seconds +[Apr 20, 2026 - 03:05:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 114 : data loading time 0.00 +[Apr 20, 2026 - 03:05:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 114 : avg forward pass time 14.22 +[Apr 20, 2026 - 03:05:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 114 : backward pass time 11.72 +[Apr 20, 2026 - 03:05:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 114 : optimizer step time 0.98 +[Apr 20, 2026 - 03:05:38 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 114-------------------- +[Apr 20, 2026 - 03:05:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0083 iter count: 1.0 +[Apr 20, 2026 - 03:05:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[Apr 20, 2026 - 03:05:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 03:05:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 03:08:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121146 43.229927 42.991425 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.434570 135.496094 135.402344 +util 71.750000 78.000000 64.000000 +[Apr 20, 2026 - 03:08:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 115 : avg iteration time 185.62 seconds +[Apr 20, 2026 - 03:08:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 115 : data loading time 0.00 +[Apr 20, 2026 - 03:08:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 115 : avg forward pass time 84.25 +[Apr 20, 2026 - 03:08:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 115 : backward pass time 8.43 +[Apr 20, 2026 - 03:08:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 115 : optimizer step time 0.33 +[Apr 20, 2026 - 03:08:44 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 115-------------------- +[Apr 20, 2026 - 03:08:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 131274.6480 iter count: 1.0 +[Apr 20, 2026 - 03:08:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.7192 iter count: 1.0 +[Apr 20, 2026 - 03:08:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 131274.6480iter count: 1.0 +[Apr 20, 2026 - 03:08:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 03:08:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.1603iter count: 1.0 +[Apr 20, 2026 - 03:08:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.3827 iter count: 1.0 +[Apr 20, 2026 - 03:08:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.5021 iter count: 1.0 +[Apr 20, 2026 - 03:08:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 183017.7031iter count: 1.0 +[Apr 20, 2026 - 03:08:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 15.8841iter count: 1.0 +[Apr 20, 2026 - 03:08:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 3282782.2500iter count: 1.0 +[Apr 20, 2026 - 03:08:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.7192iter count: 1.0 +[Apr 20, 2026 - 03:08:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 5.0361 iter count: 1.0 +[Apr 20, 2026 - 03:08:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 23677.0117iter count: 1.0 +[Apr 20, 2026 - 03:08:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 6.6689 iter count: 1.0 +[Apr 20, 2026 - 03:08:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 620214.1250iter count: 1.0 +[Apr 20, 2026 - 03:08:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 1.6093 iter count: 1.0 +[Apr 20, 2026 - 03:08:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 24.9953 iter count: 1.0 +[Apr 20, 2026 - 03:08:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 2.4161 iter count: 1.0 +[Apr 20, 2026 - 03:08:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 125.4959iter count: 1.0 +[Apr 20, 2026 - 03:08:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0076 iter count: 1.0 +[Apr 20, 2026 - 03:08:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 03:09:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121185 43.230125 42.991405 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.786621 69.917969 69.626953 +util 97.500000 99.000000 93.000000 +[Apr 20, 2026 - 03:09:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 116 : avg iteration time 57.92 seconds +[Apr 20, 2026 - 03:09:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 116 : data loading time 0.00 +[Apr 20, 2026 - 03:09:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 116 : avg forward pass time 16.89 +[Apr 20, 2026 - 03:09:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 116 : backward pass time 11.54 +[Apr 20, 2026 - 03:09:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 116 : optimizer step time 0.99 +[Apr 20, 2026 - 03:09:41 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 116-------------------- +[Apr 20, 2026 - 03:09:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0076 iter count: 1.0 +[Apr 20, 2026 - 03:09:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0076 iter count: 1.0 +[Apr 20, 2026 - 03:09:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 03:09:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 03:10:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121529 43.231499 42.991405 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.509277 72.730469 69.722656 +util 99.250000 100.000000 97.000000 +[Apr 20, 2026 - 03:10:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 117 : avg iteration time 62.19 seconds +[Apr 20, 2026 - 03:10:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 117 : data loading time 0.00 +[Apr 20, 2026 - 03:10:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 117 : avg forward pass time 18.16 +[Apr 20, 2026 - 03:10:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 117 : backward pass time 12.30 +[Apr 20, 2026 - 03:10:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 117 : optimizer step time 1.58 +[Apr 20, 2026 - 03:10:44 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 117-------------------- +[Apr 20, 2026 - 03:10:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0072 iter count: 1.0 +[Apr 20, 2026 - 03:10:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0072 iter count: 1.0 +[Apr 20, 2026 - 03:10:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 03:10:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 03:11:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121529 43.231499 42.991405 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.767090 69.878906 69.722656 +util 97.250000 100.000000 94.000000 +[Apr 20, 2026 - 03:11:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 118 : avg iteration time 58.56 seconds +[Apr 20, 2026 - 03:11:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 118 : data loading time 0.00 +[Apr 20, 2026 - 03:11:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 118 : avg forward pass time 17.22 +[Apr 20, 2026 - 03:11:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 118 : backward pass time 11.56 +[Apr 20, 2026 - 03:11:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 118 : optimizer step time 0.98 +[Apr 20, 2026 - 03:11:42 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 118-------------------- +[Apr 20, 2026 - 03:11:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0078 iter count: 1.0 +[Apr 20, 2026 - 03:11:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0078 iter count: 1.0 +[Apr 20, 2026 - 03:11:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 03:11:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 03:12:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121655 43.231983 42.991405 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.475098 72.730469 69.722656 +util 98.750000 100.000000 96.000000 +[Apr 20, 2026 - 03:12:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 119 : avg iteration time 58.89 seconds +[Apr 20, 2026 - 03:12:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 119 : data loading time 0.00 +[Apr 20, 2026 - 03:12:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 119 : avg forward pass time 16.74 +[Apr 20, 2026 - 03:12:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 119 : backward pass time 11.56 +[Apr 20, 2026 - 03:12:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 119 : optimizer step time 0.98 +[Apr 20, 2026 - 03:12:41 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 119-------------------- +[Apr 20, 2026 - 03:12:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0086 iter count: 1.0 +[Apr 20, 2026 - 03:12:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[Apr 20, 2026 - 03:12:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 03:12:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 03:15:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121267 43.230476 42.991405 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.474121 135.537109 135.402344 +util 70.250000 76.000000 62.000000 +[Apr 20, 2026 - 03:15:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 120 : avg iteration time 185.21 seconds +[Apr 20, 2026 - 03:15:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 120 : data loading time 0.00 +[Apr 20, 2026 - 03:15:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 120 : avg forward pass time 84.04 +[Apr 20, 2026 - 03:15:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 120 : backward pass time 8.37 +[Apr 20, 2026 - 03:15:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 120 : optimizer step time 0.33 +[Apr 20, 2026 - 03:15:46 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 120-------------------- +[Apr 20, 2026 - 03:15:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 46753.1160 iter count: 1.0 +[Apr 20, 2026 - 03:15:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6329 iter count: 1.0 +[Apr 20, 2026 - 03:15:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 46753.1160iter count: 1.0 +[Apr 20, 2026 - 03:15:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 03:15:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.8359iter count: 1.0 +[Apr 20, 2026 - 03:15:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.6603 iter count: 1.0 +[Apr 20, 2026 - 03:15:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.2671 iter count: 1.0 +[Apr 20, 2026 - 03:15:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 82327.5078iter count: 1.0 +[Apr 20, 2026 - 03:15:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 3.7460 iter count: 1.0 +[Apr 20, 2026 - 03:15:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 2051538.3750iter count: 1.0 +[Apr 20, 2026 - 03:15:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6329iter count: 1.0 +[Apr 20, 2026 - 03:15:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.5040 iter count: 1.0 +[Apr 20, 2026 - 03:15:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 8168.3667iter count: 1.0 +[Apr 20, 2026 - 03:15:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 1.0313 iter count: 1.0 +[Apr 20, 2026 - 03:15:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 7.8660 iter count: 1.0 +[Apr 20, 2026 - 03:15:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 3.4866 iter count: 1.0 +[Apr 20, 2026 - 03:15:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 1067.6539iter count: 1.0 +[Apr 20, 2026 - 03:15:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 1.0991 iter count: 1.0 +[Apr 20, 2026 - 03:15:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 9.0091 iter count: 1.0 +[Apr 20, 2026 - 03:15:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[Apr 20, 2026 - 03:15:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 03:16:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121492 43.231358 42.991405 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.781738 69.820312 69.666016 +util 97.750000 99.000000 96.000000 +[Apr 20, 2026 - 03:16:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 121 : avg iteration time 54.72 seconds +[Apr 20, 2026 - 03:16:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 121 : data loading time 0.00 +[Apr 20, 2026 - 03:16:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 121 : avg forward pass time 14.69 +[Apr 20, 2026 - 03:16:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 121 : backward pass time 11.52 +[Apr 20, 2026 - 03:16:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 121 : optimizer step time 1.03 +[Apr 20, 2026 - 03:16:41 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 121-------------------- +[Apr 20, 2026 - 03:16:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0069 iter count: 1.0 +[Apr 20, 2026 - 03:16:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0069 iter count: 1.0 +[Apr 20, 2026 - 03:16:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 03:16:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 03:17:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121505 43.231365 42.991405 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.864746 69.917969 69.822266 +util 95.250000 100.000000 91.000000 +[Apr 20, 2026 - 03:17:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 122 : avg iteration time 52.73 seconds +[Apr 20, 2026 - 03:17:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 122 : data loading time 0.00 +[Apr 20, 2026 - 03:17:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 122 : avg forward pass time 14.26 +[Apr 20, 2026 - 03:17:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 122 : backward pass time 11.65 +[Apr 20, 2026 - 03:17:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 122 : optimizer step time 1.00 +[Apr 20, 2026 - 03:17:34 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 122-------------------- +[Apr 20, 2026 - 03:17:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0081 iter count: 1.0 +[Apr 20, 2026 - 03:17:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[Apr 20, 2026 - 03:17:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 03:17:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 03:18:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121488 43.231358 42.991405 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.850098 69.917969 69.802734 +util 97.750000 100.000000 93.000000 +[Apr 20, 2026 - 03:18:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 123 : avg iteration time 61.11 seconds +[Apr 20, 2026 - 03:18:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 123 : data loading time 0.00 +[Apr 20, 2026 - 03:18:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 123 : avg forward pass time 17.86 +[Apr 20, 2026 - 03:18:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 123 : backward pass time 11.53 +[Apr 20, 2026 - 03:18:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 123 : optimizer step time 1.03 +[Apr 20, 2026 - 03:18:35 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 123-------------------- +[Apr 20, 2026 - 03:18:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0083 iter count: 1.0 +[Apr 20, 2026 - 03:18:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[Apr 20, 2026 - 03:18:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 03:18:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 03:19:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121494 43.231373 42.991405 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.806152 69.917969 69.666016 +util 98.000000 100.000000 96.000000 +[Apr 20, 2026 - 03:19:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 124 : avg iteration time 52.54 seconds +[Apr 20, 2026 - 03:19:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 124 : data loading time 0.00 +[Apr 20, 2026 - 03:19:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 124 : avg forward pass time 14.19 +[Apr 20, 2026 - 03:19:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 124 : backward pass time 11.59 +[Apr 20, 2026 - 03:19:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 124 : optimizer step time 0.99 +[Apr 20, 2026 - 03:19:27 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 124-------------------- +[Apr 20, 2026 - 03:19:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0078 iter count: 1.0 +[Apr 20, 2026 - 03:19:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0078 iter count: 1.0 +[Apr 20, 2026 - 03:19:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 03:19:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 03:22:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121059 43.229614 42.991409 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.458496 135.498047 135.378906 +util 72.500000 78.000000 65.000000 +[Apr 20, 2026 - 03:22:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 125 : avg iteration time 182.52 seconds +[Apr 20, 2026 - 03:22:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 125 : data loading time 0.00 +[Apr 20, 2026 - 03:22:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 125 : avg forward pass time 82.77 +[Apr 20, 2026 - 03:22:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 125 : backward pass time 8.35 +[Apr 20, 2026 - 03:22:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 125 : optimizer step time 0.34 +[Apr 20, 2026 - 03:22:30 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 125-------------------- +[Apr 20, 2026 - 03:22:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 3790313.5540 iter count: 1.0 +[Apr 20, 2026 - 03:22:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.7127 iter count: 1.0 +[Apr 20, 2026 - 03:22:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 3790313.5540iter count: 1.0 +[Apr 20, 2026 - 03:22:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 03:22:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.8917iter count: 1.0 +[Apr 20, 2026 - 03:22:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.6681 iter count: 1.0 +[Apr 20, 2026 - 03:22:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 9.3475 iter count: 1.0 +[Apr 20, 2026 - 03:22:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 4247352.0000iter count: 1.0 +[Apr 20, 2026 - 03:22:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 3.8049 iter count: 1.0 +[Apr 20, 2026 - 03:22:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 131561936.0000iter count: 1.0 +[Apr 20, 2026 - 03:22:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.7127iter count: 1.0 +[Apr 20, 2026 - 03:22:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.5992 iter count: 1.0 +[Apr 20, 2026 - 03:22:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 1337.2162iter count: 1.0 +[Apr 20, 2026 - 03:22:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.6129 iter count: 1.0 +[Apr 20, 2026 - 03:22:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 1374.3480iter count: 1.0 +[Apr 20, 2026 - 03:22:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.8182 iter count: 1.0 +[Apr 20, 2026 - 03:22:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 15311.0732iter count: 1.0 +[Apr 20, 2026 - 03:22:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 5.3282 iter count: 1.0 +[Apr 20, 2026 - 03:22:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 42461.8203iter count: 1.0 +[Apr 20, 2026 - 03:22:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0074 iter count: 1.0 +[Apr 20, 2026 - 03:22:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 03:23:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121108 43.229832 42.991409 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.387207 72.302734 69.703125 +util 98.000000 99.000000 96.000000 +[Apr 20, 2026 - 03:23:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 126 : avg iteration time 54.29 seconds +[Apr 20, 2026 - 03:23:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 126 : data loading time 0.00 +[Apr 20, 2026 - 03:23:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 126 : avg forward pass time 14.89 +[Apr 20, 2026 - 03:23:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 126 : backward pass time 11.95 +[Apr 20, 2026 - 03:23:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 126 : optimizer step time 0.99 +[Apr 20, 2026 - 03:23:24 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 126-------------------- +[Apr 20, 2026 - 03:23:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0078 iter count: 1.0 +[Apr 20, 2026 - 03:23:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0078 iter count: 1.0 +[Apr 20, 2026 - 03:23:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 03:23:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 03:24:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121410 43.231037 42.991409 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.703613 69.800781 69.490234 +util 98.750000 100.000000 97.000000 +[Apr 20, 2026 - 03:24:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 127 : avg iteration time 56.07 seconds +[Apr 20, 2026 - 03:24:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 127 : data loading time 0.00 +[Apr 20, 2026 - 03:24:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 127 : avg forward pass time 15.39 +[Apr 20, 2026 - 03:24:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 127 : backward pass time 11.56 +[Apr 20, 2026 - 03:24:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 127 : optimizer step time 0.98 +[Apr 20, 2026 - 03:24:20 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 127-------------------- +[Apr 20, 2026 - 03:24:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0088 iter count: 1.0 +[Apr 20, 2026 - 03:24:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[Apr 20, 2026 - 03:24:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 03:24:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 03:25:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121415 43.231037 42.991409 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.728027 69.820312 69.470703 +util 96.750000 100.000000 92.000000 +[Apr 20, 2026 - 03:25:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 128 : avg iteration time 50.86 seconds +[Apr 20, 2026 - 03:25:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 128 : data loading time 0.00 +[Apr 20, 2026 - 03:25:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 128 : avg forward pass time 12.76 +[Apr 20, 2026 - 03:25:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 128 : backward pass time 11.77 +[Apr 20, 2026 - 03:25:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 128 : optimizer step time 1.49 +[Apr 20, 2026 - 03:25:11 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 128-------------------- +[Apr 20, 2026 - 03:25:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0066 iter count: 1.0 +[Apr 20, 2026 - 03:25:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0066 iter count: 1.0 +[Apr 20, 2026 - 03:25:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 03:25:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 03:26:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121412 43.231045 42.991409 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.465332 72.302734 69.839844 +util 96.250000 100.000000 93.000000 +[Apr 20, 2026 - 03:26:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 129 : avg iteration time 58.23 seconds +[Apr 20, 2026 - 03:26:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 129 : data loading time 0.00 +[Apr 20, 2026 - 03:26:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 129 : avg forward pass time 17.03 +[Apr 20, 2026 - 03:26:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 129 : backward pass time 11.55 +[Apr 20, 2026 - 03:26:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 129 : optimizer step time 0.99 +[Apr 20, 2026 - 03:26:09 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 129-------------------- +[Apr 20, 2026 - 03:26:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0081 iter count: 1.0 +[Apr 20, 2026 - 03:26:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[Apr 20, 2026 - 03:26:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 03:26:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 03:29:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121173 43.230076 42.991421 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.498535 135.558594 135.478516 +util 73.500000 79.000000 66.000000 +[Apr 20, 2026 - 03:29:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 130 : avg iteration time 178.96 seconds +[Apr 20, 2026 - 03:29:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 130 : data loading time 0.00 +[Apr 20, 2026 - 03:29:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 130 : avg forward pass time 80.84 +[Apr 20, 2026 - 03:29:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 130 : backward pass time 8.40 +[Apr 20, 2026 - 03:29:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 130 : optimizer step time 0.33 +[Apr 20, 2026 - 03:29:08 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 130-------------------- +[Apr 20, 2026 - 03:29:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 42907.4243 iter count: 1.0 +[Apr 20, 2026 - 03:29:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6912 iter count: 1.0 +[Apr 20, 2026 - 03:29:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 42907.4243iter count: 1.0 +[Apr 20, 2026 - 03:29:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 03:29:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.5220iter count: 1.0 +[Apr 20, 2026 - 03:29:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.7772 iter count: 1.0 +[Apr 20, 2026 - 03:29:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.6907 iter count: 1.0 +[Apr 20, 2026 - 03:29:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 58072.7422iter count: 1.0 +[Apr 20, 2026 - 03:29:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 4.7327 iter count: 1.0 +[Apr 20, 2026 - 03:29:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 647904.3125iter count: 1.0 +[Apr 20, 2026 - 03:29:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6912iter count: 1.0 +[Apr 20, 2026 - 03:29:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.6013 iter count: 1.0 +[Apr 20, 2026 - 03:29:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 9922.6348iter count: 1.0 +[Apr 20, 2026 - 03:29:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.5829 iter count: 1.0 +[Apr 20, 2026 - 03:29:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 9565.2461iter count: 1.0 +[Apr 20, 2026 - 03:29:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 1.6828 iter count: 1.0 +[Apr 20, 2026 - 03:29:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 28.9524 iter count: 1.0 +[Apr 20, 2026 - 03:29:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 5.3214 iter count: 1.0 +[Apr 20, 2026 - 03:29:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 41893.5508iter count: 1.0 +[Apr 20, 2026 - 03:29:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0075 iter count: 1.0 +[Apr 20, 2026 - 03:29:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 03:30:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121264 43.230453 42.991409 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.830566 69.900391 69.800781 +util 98.750000 100.000000 96.000000 +[Apr 20, 2026 - 03:30:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 131 : avg iteration time 54.55 seconds +[Apr 20, 2026 - 03:30:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 131 : data loading time 0.00 +[Apr 20, 2026 - 03:30:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 131 : avg forward pass time 14.88 +[Apr 20, 2026 - 03:30:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 131 : backward pass time 11.56 +[Apr 20, 2026 - 03:30:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 131 : optimizer step time 0.98 +[Apr 20, 2026 - 03:30:03 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 131-------------------- +[Apr 20, 2026 - 03:30:03 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0082 iter count: 1.0 +[Apr 20, 2026 - 03:30:03 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[Apr 20, 2026 - 03:30:03 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 03:30:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 03:31:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121265 43.230453 42.991409 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.767090 69.878906 69.626953 +util 97.000000 100.000000 92.000000 +[Apr 20, 2026 - 03:31:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 132 : avg iteration time 64.41 seconds +[Apr 20, 2026 - 03:31:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 132 : data loading time 0.00 +[Apr 20, 2026 - 03:31:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 132 : avg forward pass time 19.84 +[Apr 20, 2026 - 03:31:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 132 : backward pass time 12.14 +[Apr 20, 2026 - 03:31:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 132 : optimizer step time 0.97 +[Apr 20, 2026 - 03:31:07 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 132-------------------- +[Apr 20, 2026 - 03:31:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0102 iter count: 1.0 +[Apr 20, 2026 - 03:31:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[Apr 20, 2026 - 03:31:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 03:31:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 03:32:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121271 43.230453 42.991432 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.698730 69.820312 69.607422 +util 98.250000 100.000000 94.000000 +[Apr 20, 2026 - 03:32:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 133 : avg iteration time 55.64 seconds +[Apr 20, 2026 - 03:32:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 133 : data loading time 0.00 +[Apr 20, 2026 - 03:32:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 133 : avg forward pass time 15.46 +[Apr 20, 2026 - 03:32:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 133 : backward pass time 11.55 +[Apr 20, 2026 - 03:32:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 133 : optimizer step time 0.98 +[Apr 20, 2026 - 03:32:03 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 133-------------------- +[Apr 20, 2026 - 03:32:03 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0080 iter count: 1.0 +[Apr 20, 2026 - 03:32:03 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0080 iter count: 1.0 +[Apr 20, 2026 - 03:32:03 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 03:32:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 03:32:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121328 43.230709 42.991409 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.796387 69.820312 69.724609 +util 96.500000 100.000000 91.000000 +[Apr 20, 2026 - 03:32:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 134 : avg iteration time 54.10 seconds +[Apr 20, 2026 - 03:32:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 134 : data loading time 0.00 +[Apr 20, 2026 - 03:32:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 134 : avg forward pass time 14.24 +[Apr 20, 2026 - 03:32:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 134 : backward pass time 12.50 +[Apr 20, 2026 - 03:32:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 134 : optimizer step time 1.51 +[Apr 20, 2026 - 03:32:57 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 134-------------------- +[Apr 20, 2026 - 03:32:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0071 iter count: 1.0 +[Apr 20, 2026 - 03:32:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0071 iter count: 1.0 +[Apr 20, 2026 - 03:32:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 03:32:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 03:36:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.120944 43.229168 42.991409 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.493164 135.537109 135.437500 +util 71.250000 78.000000 67.000000 +[Apr 20, 2026 - 03:36:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 135 : avg iteration time 188.39 seconds +[Apr 20, 2026 - 03:36:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 135 : data loading time 0.00 +[Apr 20, 2026 - 03:36:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 135 : avg forward pass time 85.72 +[Apr 20, 2026 - 03:36:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 135 : backward pass time 8.34 +[Apr 20, 2026 - 03:36:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 135 : optimizer step time 0.33 +[Apr 20, 2026 - 03:36:05 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 135-------------------- +[Apr 20, 2026 - 03:36:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 23834.4924 iter count: 1.0 +[Apr 20, 2026 - 03:36:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6545 iter count: 1.0 +[Apr 20, 2026 - 03:36:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 23834.4924iter count: 1.0 +[Apr 20, 2026 - 03:36:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 03:36:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.7770iter count: 1.0 +[Apr 20, 2026 - 03:36:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.3463 iter count: 1.0 +[Apr 20, 2026 - 03:36:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.5514 iter count: 1.0 +[Apr 20, 2026 - 03:36:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 45783.1914iter count: 1.0 +[Apr 20, 2026 - 03:36:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 1.9988 iter count: 1.0 +[Apr 20, 2026 - 03:36:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 490313.5625iter count: 1.0 +[Apr 20, 2026 - 03:36:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6545iter count: 1.0 +[Apr 20, 2026 - 03:36:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.7948 iter count: 1.0 +[Apr 20, 2026 - 03:36:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 1977.4274iter count: 1.0 +[Apr 20, 2026 - 03:36:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 2.3557 iter count: 1.0 +[Apr 20, 2026 - 03:36:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 111.2021iter count: 1.0 +[Apr 20, 2026 - 03:36:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.5766 iter count: 1.0 +[Apr 20, 2026 - 03:36:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 9444.7783iter count: 1.0 +[Apr 20, 2026 - 03:36:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 2.1095 iter count: 1.0 +[Apr 20, 2026 - 03:36:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 67.9608 iter count: 1.0 +[Apr 20, 2026 - 03:36:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0080 iter count: 1.0 +[Apr 20, 2026 - 03:36:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 03:37:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121006 43.229416 42.991409 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.903809 69.937500 69.880859 +util 98.000000 100.000000 94.000000 +[Apr 20, 2026 - 03:37:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 136 : avg iteration time 58.12 seconds +[Apr 20, 2026 - 03:37:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 136 : data loading time 0.00 +[Apr 20, 2026 - 03:37:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 136 : avg forward pass time 16.35 +[Apr 20, 2026 - 03:37:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 136 : backward pass time 12.80 +[Apr 20, 2026 - 03:37:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 136 : optimizer step time 1.05 +[Apr 20, 2026 - 03:37:04 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 136-------------------- +[Apr 20, 2026 - 03:37:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0076 iter count: 1.0 +[Apr 20, 2026 - 03:37:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0076 iter count: 1.0 +[Apr 20, 2026 - 03:37:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 03:37:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 03:38:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121376 43.230896 42.991409 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.830566 69.878906 69.763672 +util 98.500000 100.000000 96.000000 +[Apr 20, 2026 - 03:38:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 137 : avg iteration time 60.74 seconds +[Apr 20, 2026 - 03:38:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 137 : data loading time 0.00 +[Apr 20, 2026 - 03:38:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 137 : avg forward pass time 18.13 +[Apr 20, 2026 - 03:38:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 137 : backward pass time 11.57 +[Apr 20, 2026 - 03:38:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 137 : optimizer step time 0.97 +[Apr 20, 2026 - 03:38:04 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 137-------------------- +[Apr 20, 2026 - 03:38:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0067 iter count: 1.0 +[Apr 20, 2026 - 03:38:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0067 iter count: 1.0 +[Apr 20, 2026 - 03:38:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 03:38:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 03:39:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121376 43.230896 42.991409 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.879395 69.996094 69.763672 +util 98.250000 100.000000 94.000000 +[Apr 20, 2026 - 03:39:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 138 : avg iteration time 61.45 seconds +[Apr 20, 2026 - 03:39:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 138 : data loading time 0.00 +[Apr 20, 2026 - 03:39:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 138 : avg forward pass time 18.65 +[Apr 20, 2026 - 03:39:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 138 : backward pass time 11.56 +[Apr 20, 2026 - 03:39:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 138 : optimizer step time 0.98 +[Apr 20, 2026 - 03:39:06 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 138-------------------- +[Apr 20, 2026 - 03:39:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0072 iter count: 1.0 +[Apr 20, 2026 - 03:39:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0072 iter count: 1.0 +[Apr 20, 2026 - 03:39:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 03:39:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 03:40:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121402 43.230999 42.991409 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.835449 69.878906 69.763672 +util 95.500000 100.000000 92.000000 +[Apr 20, 2026 - 03:40:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 139 : avg iteration time 55.84 seconds +[Apr 20, 2026 - 03:40:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 139 : data loading time 0.00 +[Apr 20, 2026 - 03:40:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 139 : avg forward pass time 15.20 +[Apr 20, 2026 - 03:40:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 139 : backward pass time 11.56 +[Apr 20, 2026 - 03:40:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 139 : optimizer step time 0.98 +[Apr 20, 2026 - 03:40:02 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 139-------------------- +[Apr 20, 2026 - 03:40:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0073 iter count: 1.0 +[Apr 20, 2026 - 03:40:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0073 iter count: 1.0 +[Apr 20, 2026 - 03:40:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 03:40:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 03:43:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121261 43.230427 42.991409 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.518066 135.576172 135.439453 +util 73.500000 78.000000 68.000000 +[Apr 20, 2026 - 03:43:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 140 : avg iteration time 184.92 seconds +[Apr 20, 2026 - 03:43:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 140 : data loading time 0.00 +[Apr 20, 2026 - 03:43:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 140 : avg forward pass time 83.86 +[Apr 20, 2026 - 03:43:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 140 : backward pass time 8.45 +[Apr 20, 2026 - 03:43:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 140 : optimizer step time 0.33 +[Apr 20, 2026 - 03:43:06 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 140-------------------- +[Apr 20, 2026 - 03:43:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 11228.7635 iter count: 1.0 +[Apr 20, 2026 - 03:43:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6074 iter count: 1.0 +[Apr 20, 2026 - 03:43:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 11228.7635iter count: 1.0 +[Apr 20, 2026 - 03:43:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 03:43:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.6156iter count: 1.0 +[Apr 20, 2026 - 03:43:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.8145 iter count: 1.0 +[Apr 20, 2026 - 03:43:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.1075 iter count: 1.0 +[Apr 20, 2026 - 03:43:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 23844.7500iter count: 1.0 +[Apr 20, 2026 - 03:43:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 5.0992 iter count: 1.0 +[Apr 20, 2026 - 03:43:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 201784.6406iter count: 1.0 +[Apr 20, 2026 - 03:43:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6074iter count: 1.0 +[Apr 20, 2026 - 03:43:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.2721 iter count: 1.0 +[Apr 20, 2026 - 03:43:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 5137.1924iter count: 1.0 +[Apr 20, 2026 - 03:43:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.5417 iter count: 1.0 +[Apr 20, 2026 - 03:43:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 8808.1875iter count: 1.0 +[Apr 20, 2026 - 03:43:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 3.5861 iter count: 1.0 +[Apr 20, 2026 - 03:43:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 1302.7983iter count: 1.0 +[Apr 20, 2026 - 03:43:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 3.6880 iter count: 1.0 +[Apr 20, 2026 - 03:43:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 1597.1324iter count: 1.0 +[Apr 20, 2026 - 03:43:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0079 iter count: 1.0 +[Apr 20, 2026 - 03:43:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 03:44:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121385 43.230900 42.991409 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.723145 69.878906 69.646484 +util 95.250000 99.000000 92.000000 +[Apr 20, 2026 - 03:44:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 141 : avg iteration time 54.88 seconds +[Apr 20, 2026 - 03:44:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 141 : data loading time 0.00 +[Apr 20, 2026 - 03:44:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 141 : avg forward pass time 14.82 +[Apr 20, 2026 - 03:44:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 141 : backward pass time 11.52 +[Apr 20, 2026 - 03:44:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 141 : optimizer step time 1.02 +[Apr 20, 2026 - 03:44:01 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 141-------------------- +[Apr 20, 2026 - 03:44:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0075 iter count: 1.0 +[Apr 20, 2026 - 03:44:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0075 iter count: 1.0 +[Apr 20, 2026 - 03:44:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 03:44:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 03:45:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121385 43.230900 42.991432 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.806152 69.839844 69.705078 +util 98.250000 100.000000 93.000000 +[Apr 20, 2026 - 03:45:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 142 : avg iteration time 64.68 seconds +[Apr 20, 2026 - 03:45:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 142 : data loading time 0.00 +[Apr 20, 2026 - 03:45:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 142 : avg forward pass time 20.27 +[Apr 20, 2026 - 03:45:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 142 : backward pass time 11.63 +[Apr 20, 2026 - 03:45:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 142 : optimizer step time 0.96 +[Apr 20, 2026 - 03:45:06 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 142-------------------- +[Apr 20, 2026 - 03:45:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0091 iter count: 1.0 +[Apr 20, 2026 - 03:45:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[Apr 20, 2026 - 03:45:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 03:45:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 03:45:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121395 43.230923 42.991409 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.830566 69.937500 69.705078 +util 98.250000 100.000000 94.000000 +[Apr 20, 2026 - 03:45:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 143 : avg iteration time 52.41 seconds +[Apr 20, 2026 - 03:45:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 143 : data loading time 0.00 +[Apr 20, 2026 - 03:45:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 143 : avg forward pass time 14.08 +[Apr 20, 2026 - 03:45:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 143 : backward pass time 11.55 +[Apr 20, 2026 - 03:45:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 143 : optimizer step time 0.97 +[Apr 20, 2026 - 03:45:58 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 143-------------------- +[Apr 20, 2026 - 03:45:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0079 iter count: 1.0 +[Apr 20, 2026 - 03:45:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0079 iter count: 1.0 +[Apr 20, 2026 - 03:45:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 03:45:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 03:46:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121496 43.231323 42.991432 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.771973 69.917969 69.607422 +util 98.000000 100.000000 93.000000 +[Apr 20, 2026 - 03:46:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 144 : avg iteration time 56.06 seconds +[Apr 20, 2026 - 03:46:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 144 : data loading time 0.00 +[Apr 20, 2026 - 03:46:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 144 : avg forward pass time 15.58 +[Apr 20, 2026 - 03:46:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 144 : backward pass time 12.36 +[Apr 20, 2026 - 03:46:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 144 : optimizer step time 0.99 +[Apr 20, 2026 - 03:46:55 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 144-------------------- +[Apr 20, 2026 - 03:46:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0068 iter count: 1.0 +[Apr 20, 2026 - 03:46:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0068 iter count: 1.0 +[Apr 20, 2026 - 03:46:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 03:46:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 03:50:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121091 43.229710 42.991409 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.473145 135.496094 135.419922 +util 72.000000 77.000000 66.000000 +[Apr 20, 2026 - 03:50:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 145 : avg iteration time 185.75 seconds +[Apr 20, 2026 - 03:50:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 145 : data loading time 0.00 +[Apr 20, 2026 - 03:50:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 145 : avg forward pass time 84.36 +[Apr 20, 2026 - 03:50:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 145 : backward pass time 8.43 +[Apr 20, 2026 - 03:50:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 145 : optimizer step time 0.33 +[Apr 20, 2026 - 03:50:00 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 145-------------------- +[Apr 20, 2026 - 03:50:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 13238.9847 iter count: 1.0 +[Apr 20, 2026 - 03:50:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6617 iter count: 1.0 +[Apr 20, 2026 - 03:50:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 13238.9847iter count: 1.0 +[Apr 20, 2026 - 03:50:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 03:50:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.2730iter count: 1.0 +[Apr 20, 2026 - 03:50:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.4557 iter count: 1.0 +[Apr 20, 2026 - 03:50:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.0282 iter count: 1.0 +[Apr 20, 2026 - 03:50:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 20484.8359iter count: 1.0 +[Apr 20, 2026 - 03:50:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 18.3842iter count: 1.0 +[Apr 20, 2026 - 03:50:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 172190.7812iter count: 1.0 +[Apr 20, 2026 - 03:50:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6617iter count: 1.0 +[Apr 20, 2026 - 03:50:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.7938 iter count: 1.0 +[Apr 20, 2026 - 03:50:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 14582.6445iter count: 1.0 +[Apr 20, 2026 - 03:50:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 1.4557 iter count: 1.0 +[Apr 20, 2026 - 03:50:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 18.3842 iter count: 1.0 +[Apr 20, 2026 - 03:50:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 5.0049 iter count: 1.0 +[Apr 20, 2026 - 03:50:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 22244.0195iter count: 1.0 +[Apr 20, 2026 - 03:50:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 2.6795 iter count: 1.0 +[Apr 20, 2026 - 03:50:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 212.5025iter count: 1.0 +[Apr 20, 2026 - 03:50:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0077 iter count: 1.0 +[Apr 20, 2026 - 03:50:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 03:51:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121070 43.229664 42.991409 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.791504 69.859375 69.744141 +util 97.750000 100.000000 93.000000 +[Apr 20, 2026 - 03:51:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 146 : avg iteration time 59.79 seconds +[Apr 20, 2026 - 03:51:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 146 : data loading time 0.00 +[Apr 20, 2026 - 03:51:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 146 : avg forward pass time 17.44 +[Apr 20, 2026 - 03:51:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 146 : backward pass time 11.54 +[Apr 20, 2026 - 03:51:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 146 : optimizer step time 1.02 +[Apr 20, 2026 - 03:51:00 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 146-------------------- +[Apr 20, 2026 - 03:51:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0086 iter count: 1.0 +[Apr 20, 2026 - 03:51:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[Apr 20, 2026 - 03:51:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 03:51:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 03:51:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121326 43.230667 42.991409 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.811035 69.878906 69.781250 +util 97.500000 100.000000 92.000000 +[Apr 20, 2026 - 03:51:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 147 : avg iteration time 55.38 seconds +[Apr 20, 2026 - 03:51:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 147 : data loading time 0.00 +[Apr 20, 2026 - 03:51:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 147 : avg forward pass time 15.62 +[Apr 20, 2026 - 03:51:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 147 : backward pass time 11.53 +[Apr 20, 2026 - 03:51:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 147 : optimizer step time 1.03 +[Apr 20, 2026 - 03:51:55 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 147-------------------- +[Apr 20, 2026 - 03:51:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0081 iter count: 1.0 +[Apr 20, 2026 - 03:51:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[Apr 20, 2026 - 03:51:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 03:51:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 03:52:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121734 43.232319 42.991409 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.835449 69.878906 69.783203 +util 98.750000 100.000000 96.000000 +[Apr 20, 2026 - 03:52:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 148 : avg iteration time 61.44 seconds +[Apr 20, 2026 - 03:52:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 148 : data loading time 0.00 +[Apr 20, 2026 - 03:52:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 148 : avg forward pass time 17.90 +[Apr 20, 2026 - 03:52:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 148 : backward pass time 12.11 +[Apr 20, 2026 - 03:52:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 148 : optimizer step time 1.45 +[Apr 20, 2026 - 03:52:57 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 148-------------------- +[Apr 20, 2026 - 03:52:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0069 iter count: 1.0 +[Apr 20, 2026 - 03:52:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0069 iter count: 1.0 +[Apr 20, 2026 - 03:52:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 03:52:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 03:53:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121848 43.232750 42.991432 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.830566 69.878906 69.800781 +util 98.000000 100.000000 94.000000 +[Apr 20, 2026 - 03:53:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 149 : avg iteration time 47.45 seconds +[Apr 20, 2026 - 03:53:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 149 : data loading time 0.00 +[Apr 20, 2026 - 03:53:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 149 : avg forward pass time 11.31 +[Apr 20, 2026 - 03:53:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 149 : backward pass time 12.23 +[Apr 20, 2026 - 03:53:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 149 : optimizer step time 0.99 +[Apr 20, 2026 - 03:53:44 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 149-------------------- +[Apr 20, 2026 - 03:53:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0080 iter count: 1.0 +[Apr 20, 2026 - 03:53:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0080 iter count: 1.0 +[Apr 20, 2026 - 03:53:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 03:53:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 03:56:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121186 43.230129 42.991409 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.526855 135.574219 135.439453 +util 72.250000 85.000000 62.000000 +[Apr 20, 2026 - 03:56:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 150 : avg iteration time 182.22 seconds +[Apr 20, 2026 - 03:56:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 150 : data loading time 0.00 +[Apr 20, 2026 - 03:56:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 150 : avg forward pass time 82.10 +[Apr 20, 2026 - 03:56:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 150 : backward pass time 9.31 +[Apr 20, 2026 - 03:56:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 150 : optimizer step time 0.33 +[Apr 20, 2026 - 03:56:47 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 150-------------------- +[Apr 20, 2026 - 03:56:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 35073.9531 iter count: 1.0 +[Apr 20, 2026 - 03:56:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6189 iter count: 1.0 +[Apr 20, 2026 - 03:56:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 35073.9531iter count: 1.0 +[Apr 20, 2026 - 03:56:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 03:56:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.3199iter count: 1.0 +[Apr 20, 2026 - 03:56:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.2942 iter count: 1.0 +[Apr 20, 2026 - 03:56:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.4310 iter count: 1.0 +[Apr 20, 2026 - 03:56:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 64166.8359iter count: 1.0 +[Apr 20, 2026 - 03:56:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 13.3073iter count: 1.0 +[Apr 20, 2026 - 03:56:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 385351.7500iter count: 1.0 +[Apr 20, 2026 - 03:56:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6189iter count: 1.0 +[Apr 20, 2026 - 03:56:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 2.6413 iter count: 1.0 +[Apr 20, 2026 - 03:56:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 196.8725iter count: 1.0 +[Apr 20, 2026 - 03:56:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 5.6626 iter count: 1.0 +[Apr 20, 2026 - 03:56:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 82877.1953iter count: 1.0 +[Apr 20, 2026 - 03:56:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 6.3095 iter count: 1.0 +[Apr 20, 2026 - 03:56:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 302263.5938iter count: 1.0 +[Apr 20, 2026 - 03:56:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.2931 iter count: 1.0 +[Apr 20, 2026 - 03:56:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 5357.5181iter count: 1.0 +[Apr 20, 2026 - 03:56:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0062 iter count: 1.0 +[Apr 20, 2026 - 03:56:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 03:57:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121310 43.230610 42.991409 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.460449 72.769531 69.626953 +util 98.250000 100.000000 95.000000 +[Apr 20, 2026 - 03:57:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 151 : avg iteration time 61.99 seconds +[Apr 20, 2026 - 03:57:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 151 : data loading time 0.00 +[Apr 20, 2026 - 03:57:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 151 : avg forward pass time 18.94 +[Apr 20, 2026 - 03:57:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 151 : backward pass time 11.56 +[Apr 20, 2026 - 03:57:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 151 : optimizer step time 0.97 +[Apr 20, 2026 - 03:57:49 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 151-------------------- +[Apr 20, 2026 - 03:57:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0074 iter count: 1.0 +[Apr 20, 2026 - 03:57:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0074 iter count: 1.0 +[Apr 20, 2026 - 03:57:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 03:57:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 03:58:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121440 43.231121 42.991409 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.548340 72.730469 69.783203 +util 96.500000 100.000000 93.000000 +[Apr 20, 2026 - 03:58:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 152 : avg iteration time 50.66 seconds +[Apr 20, 2026 - 03:58:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 152 : data loading time 0.00 +[Apr 20, 2026 - 03:58:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 152 : avg forward pass time 12.69 +[Apr 20, 2026 - 03:58:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 152 : backward pass time 12.46 +[Apr 20, 2026 - 03:58:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 152 : optimizer step time 1.21 +[Apr 20, 2026 - 03:58:39 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 152-------------------- +[Apr 20, 2026 - 03:58:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0087 iter count: 1.0 +[Apr 20, 2026 - 03:58:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[Apr 20, 2026 - 03:58:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 03:58:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 03:59:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121434 43.231098 42.991432 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.528809 72.710938 69.800781 +util 96.000000 99.000000 93.000000 +[Apr 20, 2026 - 03:59:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 153 : avg iteration time 49.59 seconds +[Apr 20, 2026 - 03:59:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 153 : data loading time 0.01 +[Apr 20, 2026 - 03:59:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 153 : avg forward pass time 12.74 +[Apr 20, 2026 - 03:59:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 153 : backward pass time 11.52 +[Apr 20, 2026 - 03:59:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 153 : optimizer step time 1.02 +[Apr 20, 2026 - 03:59:29 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 153-------------------- +[Apr 20, 2026 - 03:59:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0074 iter count: 1.0 +[Apr 20, 2026 - 03:59:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0074 iter count: 1.0 +[Apr 20, 2026 - 03:59:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 03:59:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 04:00:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121446 43.231098 42.991432 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.475098 72.710938 69.587891 +util 97.250000 100.000000 92.000000 +[Apr 20, 2026 - 04:00:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 154 : avg iteration time 58.72 seconds +[Apr 20, 2026 - 04:00:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 154 : data loading time 0.00 +[Apr 20, 2026 - 04:00:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 154 : avg forward pass time 16.59 +[Apr 20, 2026 - 04:00:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 154 : backward pass time 11.60 +[Apr 20, 2026 - 04:00:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 154 : optimizer step time 1.40 +[Apr 20, 2026 - 04:00:28 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 154-------------------- +[Apr 20, 2026 - 04:00:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0087 iter count: 1.0 +[Apr 20, 2026 - 04:00:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[Apr 20, 2026 - 04:00:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 04:00:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 04:03:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121270 43.230457 42.991417 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.483887 135.519531 135.458984 +util 72.250000 77.000000 65.000000 +[Apr 20, 2026 - 04:03:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 155 : avg iteration time 197.03 seconds +[Apr 20, 2026 - 04:03:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 155 : data loading time 0.00 +[Apr 20, 2026 - 04:03:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 155 : avg forward pass time 89.98 +[Apr 20, 2026 - 04:03:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 155 : backward pass time 8.39 +[Apr 20, 2026 - 04:03:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 155 : optimizer step time 0.34 +[Apr 20, 2026 - 04:03:45 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 155-------------------- +[Apr 20, 2026 - 04:03:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 247164.3391 iter count: 1.0 +[Apr 20, 2026 - 04:03:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6669 iter count: 1.0 +[Apr 20, 2026 - 04:03:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 247164.3391iter count: 1.0 +[Apr 20, 2026 - 04:03:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 04:03:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.5143iter count: 1.0 +[Apr 20, 2026 - 04:03:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.6187 iter count: 1.0 +[Apr 20, 2026 - 04:03:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.8054 iter count: 1.0 +[Apr 20, 2026 - 04:03:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 327104.5625iter count: 1.0 +[Apr 20, 2026 - 04:03:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 25.4698iter count: 1.0 +[Apr 20, 2026 - 04:03:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 6020802.5000iter count: 1.0 +[Apr 20, 2026 - 04:03:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6669iter count: 1.0 +[Apr 20, 2026 - 04:03:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.1387 iter count: 1.0 +[Apr 20, 2026 - 04:03:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 3934.1970iter count: 1.0 +[Apr 20, 2026 - 04:03:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 5.2102 iter count: 1.0 +[Apr 20, 2026 - 04:03:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 33539.4258iter count: 1.0 +[Apr 20, 2026 - 04:03:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 3.3806 iter count: 1.0 +[Apr 20, 2026 - 04:03:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 863.6252iter count: 1.0 +[Apr 20, 2026 - 04:03:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 5.8269 iter count: 1.0 +[Apr 20, 2026 - 04:03:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 115117.5938iter count: 1.0 +[Apr 20, 2026 - 04:03:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[Apr 20, 2026 - 04:03:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 04:04:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121286 43.230438 42.991432 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.830566 69.917969 69.646484 +util 98.250000 100.000000 94.000000 +[Apr 20, 2026 - 04:04:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 156 : avg iteration time 65.47 seconds +[Apr 20, 2026 - 04:04:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 156 : data loading time 0.00 +[Apr 20, 2026 - 04:04:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 156 : avg forward pass time 19.90 +[Apr 20, 2026 - 04:04:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 156 : backward pass time 11.64 +[Apr 20, 2026 - 04:04:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 156 : optimizer step time 1.42 +[Apr 20, 2026 - 04:04:50 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 156-------------------- +[Apr 20, 2026 - 04:04:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0105 iter count: 1.0 +[Apr 20, 2026 - 04:04:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[Apr 20, 2026 - 04:04:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 04:04:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 04:05:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121264 43.230438 42.991409 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.791504 69.839844 69.646484 +util 96.750000 100.000000 91.000000 +[Apr 20, 2026 - 04:05:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 157 : avg iteration time 57.90 seconds +[Apr 20, 2026 - 04:05:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 157 : data loading time 0.00 +[Apr 20, 2026 - 04:05:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 157 : avg forward pass time 16.91 +[Apr 20, 2026 - 04:05:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 157 : backward pass time 11.55 +[Apr 20, 2026 - 04:05:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 157 : optimizer step time 0.98 +[Apr 20, 2026 - 04:05:48 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 157-------------------- +[Apr 20, 2026 - 04:05:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0087 iter count: 1.0 +[Apr 20, 2026 - 04:05:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[Apr 20, 2026 - 04:05:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 04:05:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 04:06:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121340 43.230633 42.991432 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.811035 69.976562 69.626953 +util 97.000000 100.000000 91.000000 +[Apr 20, 2026 - 04:06:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 158 : avg iteration time 59.13 seconds +[Apr 20, 2026 - 04:06:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 158 : data loading time 0.00 +[Apr 20, 2026 - 04:06:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 158 : avg forward pass time 16.51 +[Apr 20, 2026 - 04:06:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 158 : backward pass time 12.30 +[Apr 20, 2026 - 04:06:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 158 : optimizer step time 1.41 +[Apr 20, 2026 - 04:06:47 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 158-------------------- +[Apr 20, 2026 - 04:06:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0102 iter count: 1.0 +[Apr 20, 2026 - 04:06:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[Apr 20, 2026 - 04:06:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 04:06:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 04:07:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121313 43.230633 42.991413 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.830566 69.976562 69.666016 +util 98.000000 100.000000 96.000000 +[Apr 20, 2026 - 04:07:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 159 : avg iteration time 52.58 seconds +[Apr 20, 2026 - 04:07:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 159 : data loading time 0.00 +[Apr 20, 2026 - 04:07:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 159 : avg forward pass time 14.26 +[Apr 20, 2026 - 04:07:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 159 : backward pass time 11.53 +[Apr 20, 2026 - 04:07:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 159 : optimizer step time 1.00 +[Apr 20, 2026 - 04:07:40 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 159-------------------- +[Apr 20, 2026 - 04:07:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0074 iter count: 1.0 +[Apr 20, 2026 - 04:07:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0074 iter count: 1.0 +[Apr 20, 2026 - 04:07:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 04:07:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 04:10:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121000 43.229336 42.991436 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.448730 135.480469 135.400391 +util 68.250000 75.000000 64.000000 +[Apr 20, 2026 - 04:10:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 160 : avg iteration time 186.19 seconds +[Apr 20, 2026 - 04:10:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 160 : data loading time 0.00 +[Apr 20, 2026 - 04:10:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 160 : avg forward pass time 84.56 +[Apr 20, 2026 - 04:10:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 160 : backward pass time 8.42 +[Apr 20, 2026 - 04:10:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 160 : optimizer step time 0.35 +[Apr 20, 2026 - 04:10:46 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 160-------------------- +[Apr 20, 2026 - 04:10:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 43363.6841 iter count: 1.0 +[Apr 20, 2026 - 04:10:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6629 iter count: 1.0 +[Apr 20, 2026 - 04:10:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 43363.6841iter count: 1.0 +[Apr 20, 2026 - 04:10:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 04:10:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.6425iter count: 1.0 +[Apr 20, 2026 - 04:10:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.7307 iter count: 1.0 +[Apr 20, 2026 - 04:10:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.0273 iter count: 1.0 +[Apr 20, 2026 - 04:10:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 53685.1914iter count: 1.0 +[Apr 20, 2026 - 04:10:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 4.3119 iter count: 1.0 +[Apr 20, 2026 - 04:10:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 1270076.0000iter count: 1.0 +[Apr 20, 2026 - 04:10:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6629iter count: 1.0 +[Apr 20, 2026 - 04:10:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.6853 iter count: 1.0 +[Apr 20, 2026 - 04:10:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 1588.5135iter count: 1.0 +[Apr 20, 2026 - 04:10:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.8531 iter count: 1.0 +[Apr 20, 2026 - 04:10:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 16419.0859iter count: 1.0 +[Apr 20, 2026 - 04:10:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 2.5399 iter count: 1.0 +[Apr 20, 2026 - 04:10:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 160.7388iter count: 1.0 +[Apr 20, 2026 - 04:10:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 3.7925 iter count: 1.0 +[Apr 20, 2026 - 04:10:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 1968.2571iter count: 1.0 +[Apr 20, 2026 - 04:10:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0076 iter count: 1.0 +[Apr 20, 2026 - 04:10:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 04:11:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121063 43.229595 42.991417 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.362793 72.029297 69.800781 +util 96.250000 100.000000 93.000000 +[Apr 20, 2026 - 04:11:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 161 : avg iteration time 62.97 seconds +[Apr 20, 2026 - 04:11:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 161 : data loading time 0.00 +[Apr 20, 2026 - 04:11:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 161 : avg forward pass time 18.71 +[Apr 20, 2026 - 04:11:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 161 : backward pass time 12.41 +[Apr 20, 2026 - 04:11:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 161 : optimizer step time 1.32 +[Apr 20, 2026 - 04:11:49 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 161-------------------- +[Apr 20, 2026 - 04:11:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0077 iter count: 1.0 +[Apr 20, 2026 - 04:11:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0077 iter count: 1.0 +[Apr 20, 2026 - 04:11:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 04:11:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 04:12:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121121 43.229836 42.991417 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.411621 71.873047 69.859375 +util 98.000000 100.000000 93.000000 +[Apr 20, 2026 - 04:12:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 162 : avg iteration time 60.67 seconds +[Apr 20, 2026 - 04:12:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 162 : data loading time 0.00 +[Apr 20, 2026 - 04:12:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 162 : avg forward pass time 18.31 +[Apr 20, 2026 - 04:12:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 162 : backward pass time 11.56 +[Apr 20, 2026 - 04:12:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 162 : optimizer step time 0.98 +[Apr 20, 2026 - 04:12:49 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 162-------------------- +[Apr 20, 2026 - 04:12:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0097 iter count: 1.0 +[Apr 20, 2026 - 04:12:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[Apr 20, 2026 - 04:12:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 04:12:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 04:13:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121115 43.229836 42.991417 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.328613 71.892578 69.800781 +util 98.500000 100.000000 95.000000 +[Apr 20, 2026 - 04:13:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 163 : avg iteration time 58.62 seconds +[Apr 20, 2026 - 04:13:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 163 : data loading time 0.00 +[Apr 20, 2026 - 04:13:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 163 : avg forward pass time 16.99 +[Apr 20, 2026 - 04:13:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 163 : backward pass time 12.10 +[Apr 20, 2026 - 04:13:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 163 : optimizer step time 0.99 +[Apr 20, 2026 - 04:13:48 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 163-------------------- +[Apr 20, 2026 - 04:13:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0083 iter count: 1.0 +[Apr 20, 2026 - 04:13:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[Apr 20, 2026 - 04:13:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 04:13:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 04:14:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121355 43.230782 42.991417 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.387207 71.912109 69.878906 +util 96.250000 99.000000 91.000000 +[Apr 20, 2026 - 04:14:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 164 : avg iteration time 46.36 seconds +[Apr 20, 2026 - 04:14:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 164 : data loading time 0.00 +[Apr 20, 2026 - 04:14:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 164 : avg forward pass time 11.14 +[Apr 20, 2026 - 04:14:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 164 : backward pass time 11.54 +[Apr 20, 2026 - 04:14:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 164 : optimizer step time 0.98 +[Apr 20, 2026 - 04:14:34 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 164-------------------- +[Apr 20, 2026 - 04:14:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0079 iter count: 1.0 +[Apr 20, 2026 - 04:14:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0079 iter count: 1.0 +[Apr 20, 2026 - 04:14:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 04:14:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 04:17:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121115 43.229790 42.991417 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.459473 135.478516 135.419922 +util 72.750000 78.000000 67.000000 +[Apr 20, 2026 - 04:17:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 165 : avg iteration time 190.46 seconds +[Apr 20, 2026 - 04:17:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 165 : data loading time 0.00 +[Apr 20, 2026 - 04:17:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 165 : avg forward pass time 86.54 +[Apr 20, 2026 - 04:17:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 165 : backward pass time 8.38 +[Apr 20, 2026 - 04:17:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 165 : optimizer step time 0.33 +[Apr 20, 2026 - 04:17:45 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 165-------------------- +[Apr 20, 2026 - 04:17:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 67916.9006 iter count: 1.0 +[Apr 20, 2026 - 04:17:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6588 iter count: 1.0 +[Apr 20, 2026 - 04:17:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 67916.9006iter count: 1.0 +[Apr 20, 2026 - 04:17:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 04:17:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.5101iter count: 1.0 +[Apr 20, 2026 - 04:17:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.8904 iter count: 1.0 +[Apr 20, 2026 - 04:17:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.5434 iter count: 1.0 +[Apr 20, 2026 - 04:17:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 157780.7656iter count: 1.0 +[Apr 20, 2026 - 04:17:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 43.8478iter count: 1.0 +[Apr 20, 2026 - 04:17:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 3565598.7500iter count: 1.0 +[Apr 20, 2026 - 04:17:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6588iter count: 1.0 +[Apr 20, 2026 - 04:17:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.1677 iter count: 1.0 +[Apr 20, 2026 - 04:17:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 4169.0498iter count: 1.0 +[Apr 20, 2026 - 04:17:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 5.4968 iter count: 1.0 +[Apr 20, 2026 - 04:17:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 59497.8984iter count: 1.0 +[Apr 20, 2026 - 04:17:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 3.7747 iter count: 1.0 +[Apr 20, 2026 - 04:17:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 1899.4474iter count: 1.0 +[Apr 20, 2026 - 04:17:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 1.8904 iter count: 1.0 +[Apr 20, 2026 - 04:17:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 43.8478 iter count: 1.0 +[Apr 20, 2026 - 04:17:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[Apr 20, 2026 - 04:17:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 04:18:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121104 43.229790 42.991421 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.830566 69.900391 69.800781 +util 95.500000 99.000000 92.000000 +[Apr 20, 2026 - 04:18:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 166 : avg iteration time 65.71 seconds +[Apr 20, 2026 - 04:18:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 166 : data loading time 0.00 +[Apr 20, 2026 - 04:18:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 166 : avg forward pass time 20.25 +[Apr 20, 2026 - 04:18:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 166 : backward pass time 11.53 +[Apr 20, 2026 - 04:18:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 166 : optimizer step time 1.04 +[Apr 20, 2026 - 04:18:51 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 166-------------------- +[Apr 20, 2026 - 04:18:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0090 iter count: 1.0 +[Apr 20, 2026 - 04:18:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[Apr 20, 2026 - 04:18:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 04:18:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 04:19:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121104 43.229790 42.991421 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.913574 69.996094 69.800781 +util 97.250000 100.000000 92.000000 +[Apr 20, 2026 - 04:19:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 167 : avg iteration time 63.51 seconds +[Apr 20, 2026 - 04:19:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 167 : data loading time 0.00 +[Apr 20, 2026 - 04:19:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 167 : avg forward pass time 19.73 +[Apr 20, 2026 - 04:19:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 167 : backward pass time 11.56 +[Apr 20, 2026 - 04:19:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 167 : optimizer step time 0.97 +[Apr 20, 2026 - 04:19:54 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 167-------------------- +[Apr 20, 2026 - 04:19:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0098 iter count: 1.0 +[Apr 20, 2026 - 04:19:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[Apr 20, 2026 - 04:19:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 04:19:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 04:20:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121104 43.229790 42.991421 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.728027 69.820312 69.683594 +util 97.250000 100.000000 92.000000 +[Apr 20, 2026 - 04:20:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 168 : avg iteration time 62.00 seconds +[Apr 20, 2026 - 04:20:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 168 : data loading time 0.00 +[Apr 20, 2026 - 04:20:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 168 : avg forward pass time 18.31 +[Apr 20, 2026 - 04:20:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 168 : backward pass time 12.74 +[Apr 20, 2026 - 04:20:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 168 : optimizer step time 1.05 +[Apr 20, 2026 - 04:20:56 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 168-------------------- +[Apr 20, 2026 - 04:20:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0088 iter count: 1.0 +[Apr 20, 2026 - 04:20:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[Apr 20, 2026 - 04:20:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 04:20:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 04:21:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121112 43.229797 42.991421 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.762207 69.859375 69.683594 +util 98.500000 100.000000 95.000000 +[Apr 20, 2026 - 04:21:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 169 : avg iteration time 49.17 seconds +[Apr 20, 2026 - 04:21:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 169 : data loading time 0.00 +[Apr 20, 2026 - 04:21:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 169 : avg forward pass time 12.56 +[Apr 20, 2026 - 04:21:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 169 : backward pass time 11.55 +[Apr 20, 2026 - 04:21:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 169 : optimizer step time 0.98 +[Apr 20, 2026 - 04:21:45 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 169-------------------- +[Apr 20, 2026 - 04:21:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0069 iter count: 1.0 +[Apr 20, 2026 - 04:21:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0069 iter count: 1.0 +[Apr 20, 2026 - 04:21:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 04:21:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 04:24:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.120973 43.229252 42.991432 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.453613 135.476562 135.439453 +util 79.750000 96.000000 69.000000 +[Apr 20, 2026 - 04:24:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 170 : avg iteration time 183.08 seconds +[Apr 20, 2026 - 04:24:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 170 : data loading time 0.00 +[Apr 20, 2026 - 04:24:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 170 : avg forward pass time 82.52 +[Apr 20, 2026 - 04:24:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 170 : backward pass time 9.42 +[Apr 20, 2026 - 04:24:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 170 : optimizer step time 0.33 +[Apr 20, 2026 - 04:24:48 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 170-------------------- +[Apr 20, 2026 - 04:24:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 188800.0522 iter count: 1.0 +[Apr 20, 2026 - 04:24:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6837 iter count: 1.0 +[Apr 20, 2026 - 04:24:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 188800.0522iter count: 1.0 +[Apr 20, 2026 - 04:24:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 04:24:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.1378iter count: 1.0 +[Apr 20, 2026 - 04:24:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.6149 iter count: 1.0 +[Apr 20, 2026 - 04:24:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 8.0538 iter count: 1.0 +[Apr 20, 2026 - 04:24:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 360394.0625iter count: 1.0 +[Apr 20, 2026 - 04:24:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 3.4208 iter count: 1.0 +[Apr 20, 2026 - 04:24:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 9894679.0000iter count: 1.0 +[Apr 20, 2026 - 04:24:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6837iter count: 1.0 +[Apr 20, 2026 - 04:24:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 8.0538 iter count: 1.0 +[Apr 20, 2026 - 04:24:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 9894679.0000iter count: 1.0 +[Apr 20, 2026 - 04:24:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 5.9322 iter count: 1.0 +[Apr 20, 2026 - 04:24:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 142128.4844iter count: 1.0 +[Apr 20, 2026 - 04:24:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.3838 iter count: 1.0 +[Apr 20, 2026 - 04:24:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 6422.1948iter count: 1.0 +[Apr 20, 2026 - 04:24:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 3.9219 iter count: 1.0 +[Apr 20, 2026 - 04:24:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 2549.7727iter count: 1.0 +[Apr 20, 2026 - 04:24:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0068 iter count: 1.0 +[Apr 20, 2026 - 04:24:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 04:25:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121108 43.229771 42.991432 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.811035 69.880859 69.781250 +util 97.250000 100.000000 92.000000 +[Apr 20, 2026 - 04:25:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 171 : avg iteration time 64.57 seconds +[Apr 20, 2026 - 04:25:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 171 : data loading time 0.00 +[Apr 20, 2026 - 04:25:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 171 : avg forward pass time 20.20 +[Apr 20, 2026 - 04:25:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 171 : backward pass time 11.62 +[Apr 20, 2026 - 04:25:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 171 : optimizer step time 1.00 +[Apr 20, 2026 - 04:25:53 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 171-------------------- +[Apr 20, 2026 - 04:25:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0096 iter count: 1.0 +[Apr 20, 2026 - 04:25:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[Apr 20, 2026 - 04:25:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 04:25:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 04:26:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121104 43.229767 42.991432 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.869629 69.880859 69.859375 +util 95.000000 99.000000 92.000000 +[Apr 20, 2026 - 04:26:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 172 : avg iteration time 53.85 seconds +[Apr 20, 2026 - 04:26:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 172 : data loading time 0.00 +[Apr 20, 2026 - 04:26:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 172 : avg forward pass time 14.21 +[Apr 20, 2026 - 04:26:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 172 : backward pass time 12.80 +[Apr 20, 2026 - 04:26:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 172 : optimizer step time 1.06 +[Apr 20, 2026 - 04:26:47 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 172-------------------- +[Apr 20, 2026 - 04:26:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0077 iter count: 1.0 +[Apr 20, 2026 - 04:26:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0077 iter count: 1.0 +[Apr 20, 2026 - 04:26:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 04:26:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 04:27:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121163 43.230015 42.991432 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.884277 69.919922 69.859375 +util 96.500000 100.000000 92.000000 +[Apr 20, 2026 - 04:27:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 173 : avg iteration time 57.86 seconds +[Apr 20, 2026 - 04:27:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 173 : data loading time 0.00 +[Apr 20, 2026 - 04:27:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 173 : avg forward pass time 16.73 +[Apr 20, 2026 - 04:27:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 173 : backward pass time 11.56 +[Apr 20, 2026 - 04:27:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 173 : optimizer step time 0.98 +[Apr 20, 2026 - 04:27:45 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 173-------------------- +[Apr 20, 2026 - 04:27:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0067 iter count: 1.0 +[Apr 20, 2026 - 04:27:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0067 iter count: 1.0 +[Apr 20, 2026 - 04:27:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 04:27:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 04:28:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121289 43.230518 42.991432 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.767090 69.820312 69.646484 +util 96.500000 100.000000 93.000000 +[Apr 20, 2026 - 04:28:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 174 : avg iteration time 61.96 seconds +[Apr 20, 2026 - 04:28:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 174 : data loading time 0.00 +[Apr 20, 2026 - 04:28:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 174 : avg forward pass time 18.48 +[Apr 20, 2026 - 04:28:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 174 : backward pass time 12.42 +[Apr 20, 2026 - 04:28:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 174 : optimizer step time 1.01 +[Apr 20, 2026 - 04:28:47 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 174-------------------- +[Apr 20, 2026 - 04:28:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0085 iter count: 1.0 +[Apr 20, 2026 - 04:28:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[Apr 20, 2026 - 04:28:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 04:28:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 04:31:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121069 43.229603 42.991436 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.483887 135.537109 135.419922 +util 72.750000 78.000000 67.000000 +[Apr 20, 2026 - 04:31:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 175 : avg iteration time 186.90 seconds +[Apr 20, 2026 - 04:31:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 175 : data loading time 0.00 +[Apr 20, 2026 - 04:31:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 175 : avg forward pass time 84.89 +[Apr 20, 2026 - 04:31:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 175 : backward pass time 8.42 +[Apr 20, 2026 - 04:31:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 175 : optimizer step time 0.33 +[Apr 20, 2026 - 04:31:54 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 175-------------------- +[Apr 20, 2026 - 04:31:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 278347.3203 iter count: 1.0 +[Apr 20, 2026 - 04:31:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5802 iter count: 1.0 +[Apr 20, 2026 - 04:31:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 278347.3203iter count: 1.0 +[Apr 20, 2026 - 04:31:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 04:31:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.5383iter count: 1.0 +[Apr 20, 2026 - 04:31:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.4126 iter count: 1.0 +[Apr 20, 2026 - 04:31:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 8.0724 iter count: 1.0 +[Apr 20, 2026 - 04:31:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 487482.5938iter count: 1.0 +[Apr 20, 2026 - 04:31:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 2.2824 iter count: 1.0 +[Apr 20, 2026 - 04:31:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 10270897.0000iter count: 1.0 +[Apr 20, 2026 - 04:31:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5802iter count: 1.0 +[Apr 20, 2026 - 04:31:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 5.1703 iter count: 1.0 +[Apr 20, 2026 - 04:31:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 30967.2910iter count: 1.0 +[Apr 20, 2026 - 04:31:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.6314 iter count: 1.0 +[Apr 20, 2026 - 04:31:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 1426.1060iter count: 1.0 +[Apr 20, 2026 - 04:31:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 5.3536 iter count: 1.0 +[Apr 20, 2026 - 04:31:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 44676.6719iter count: 1.0 +[Apr 20, 2026 - 04:31:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 2.6599 iter count: 1.0 +[Apr 20, 2026 - 04:31:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 204.3425iter count: 1.0 +[Apr 20, 2026 - 04:31:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0072 iter count: 1.0 +[Apr 20, 2026 - 04:31:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 04:32:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121073 43.229603 42.991436 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.835449 69.898438 69.781250 +util 96.250000 99.000000 93.000000 +[Apr 20, 2026 - 04:32:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 176 : avg iteration time 56.90 seconds +[Apr 20, 2026 - 04:32:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 176 : data loading time 0.00 +[Apr 20, 2026 - 04:32:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 176 : avg forward pass time 16.39 +[Apr 20, 2026 - 04:32:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 176 : backward pass time 11.55 +[Apr 20, 2026 - 04:32:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 176 : optimizer step time 1.00 +[Apr 20, 2026 - 04:32:50 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 176-------------------- +[Apr 20, 2026 - 04:32:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0080 iter count: 1.0 +[Apr 20, 2026 - 04:32:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0080 iter count: 1.0 +[Apr 20, 2026 - 04:32:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 04:32:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 04:33:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121375 43.230827 42.991436 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.845215 69.878906 69.802734 +util 97.250000 100.000000 92.000000 +[Apr 20, 2026 - 04:33:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 177 : avg iteration time 64.93 seconds +[Apr 20, 2026 - 04:33:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 177 : data loading time 0.00 +[Apr 20, 2026 - 04:33:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 177 : avg forward pass time 19.34 +[Apr 20, 2026 - 04:33:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 177 : backward pass time 12.20 +[Apr 20, 2026 - 04:33:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 177 : optimizer step time 1.58 +[Apr 20, 2026 - 04:33:55 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 177-------------------- +[Apr 20, 2026 - 04:33:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0079 iter count: 1.0 +[Apr 20, 2026 - 04:33:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0079 iter count: 1.0 +[Apr 20, 2026 - 04:33:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 04:33:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 04:34:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121383 43.230835 42.991436 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.859863 69.937500 69.783203 +util 96.500000 99.000000 94.000000 +[Apr 20, 2026 - 04:34:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 178 : avg iteration time 55.48 seconds +[Apr 20, 2026 - 04:34:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 178 : data loading time 0.00 +[Apr 20, 2026 - 04:34:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 178 : avg forward pass time 15.69 +[Apr 20, 2026 - 04:34:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 178 : backward pass time 11.55 +[Apr 20, 2026 - 04:34:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 178 : optimizer step time 1.00 +[Apr 20, 2026 - 04:34:51 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 178-------------------- +[Apr 20, 2026 - 04:34:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0068 iter count: 1.0 +[Apr 20, 2026 - 04:34:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0068 iter count: 1.0 +[Apr 20, 2026 - 04:34:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 04:34:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 04:35:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121436 43.231071 42.991436 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.869629 69.898438 69.820312 +util 99.000000 100.000000 96.000000 +[Apr 20, 2026 - 04:35:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 179 : avg iteration time 61.63 seconds +[Apr 20, 2026 - 04:35:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 179 : data loading time 0.00 +[Apr 20, 2026 - 04:35:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 179 : avg forward pass time 17.98 +[Apr 20, 2026 - 04:35:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 179 : backward pass time 11.75 +[Apr 20, 2026 - 04:35:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 179 : optimizer step time 1.47 +[Apr 20, 2026 - 04:35:52 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 179-------------------- +[Apr 20, 2026 - 04:35:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0068 iter count: 1.0 +[Apr 20, 2026 - 04:35:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0068 iter count: 1.0 +[Apr 20, 2026 - 04:35:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 04:35:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 04:38:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121595 43.231056 42.991665 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.473145 135.498047 135.437500 +util 69.500000 74.000000 67.000000 +[Apr 20, 2026 - 04:38:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 180 : avg iteration time 178.01 seconds +[Apr 20, 2026 - 04:38:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 180 : data loading time 0.00 +[Apr 20, 2026 - 04:38:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 180 : avg forward pass time 80.42 +[Apr 20, 2026 - 04:38:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 180 : backward pass time 8.39 +[Apr 20, 2026 - 04:38:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 180 : optimizer step time 0.33 +[Apr 20, 2026 - 04:38:50 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 180-------------------- +[Apr 20, 2026 - 04:38:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 67392.8618 iter count: 1.0 +[Apr 20, 2026 - 04:38:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6547 iter count: 1.0 +[Apr 20, 2026 - 04:38:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 67392.8618iter count: 1.0 +[Apr 20, 2026 - 04:38:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 04:38:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.9040iter count: 1.0 +[Apr 20, 2026 - 04:38:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.4623 iter count: 1.0 +[Apr 20, 2026 - 04:38:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.9659 iter count: 1.0 +[Apr 20, 2026 - 04:38:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 90890.5000iter count: 1.0 +[Apr 20, 2026 - 04:38:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 2.5206 iter count: 1.0 +[Apr 20, 2026 - 04:38:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 1123253.3750iter count: 1.0 +[Apr 20, 2026 - 04:38:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6547iter count: 1.0 +[Apr 20, 2026 - 04:38:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 2.9860 iter count: 1.0 +[Apr 20, 2026 - 04:38:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 392.3224iter count: 1.0 +[Apr 20, 2026 - 04:38:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.0467 iter count: 1.0 +[Apr 20, 2026 - 04:38:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 442.9539iter count: 1.0 +[Apr 20, 2026 - 04:38:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 1.8351 iter count: 1.0 +[Apr 20, 2026 - 04:38:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 39.2616 iter count: 1.0 +[Apr 20, 2026 - 04:38:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.1425 iter count: 1.0 +[Apr 20, 2026 - 04:38:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 3964.0928iter count: 1.0 +[Apr 20, 2026 - 04:38:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0068 iter count: 1.0 +[Apr 20, 2026 - 04:38:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 04:39:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121593 43.231056 42.991657 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.815918 69.841797 69.800781 +util 97.000000 100.000000 91.000000 +[Apr 20, 2026 - 04:39:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 181 : avg iteration time 66.07 seconds +[Apr 20, 2026 - 04:39:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 181 : data loading time 0.00 +[Apr 20, 2026 - 04:39:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 181 : avg forward pass time 20.86 +[Apr 20, 2026 - 04:39:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 181 : backward pass time 11.78 +[Apr 20, 2026 - 04:39:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 181 : optimizer step time 0.99 +[Apr 20, 2026 - 04:39:57 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 181-------------------- +[Apr 20, 2026 - 04:39:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0102 iter count: 1.0 +[Apr 20, 2026 - 04:39:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[Apr 20, 2026 - 04:39:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 04:39:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 04:40:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121593 43.231056 42.991657 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.806152 69.880859 69.781250 +util 97.000000 99.000000 92.000000 +[Apr 20, 2026 - 04:40:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 182 : avg iteration time 54.94 seconds +[Apr 20, 2026 - 04:40:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 182 : data loading time 0.00 +[Apr 20, 2026 - 04:40:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 182 : avg forward pass time 15.34 +[Apr 20, 2026 - 04:40:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 182 : backward pass time 11.57 +[Apr 20, 2026 - 04:40:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 182 : optimizer step time 1.03 +[Apr 20, 2026 - 04:40:51 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 182-------------------- +[Apr 20, 2026 - 04:40:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0107 iter count: 1.0 +[Apr 20, 2026 - 04:40:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[Apr 20, 2026 - 04:40:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 04:40:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 04:41:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121600 43.231056 42.991680 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.894043 69.957031 69.783203 +util 98.000000 99.000000 95.000000 +[Apr 20, 2026 - 04:41:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 183 : avg iteration time 53.84 seconds +[Apr 20, 2026 - 04:41:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 183 : data loading time 0.00 +[Apr 20, 2026 - 04:41:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 183 : avg forward pass time 14.25 +[Apr 20, 2026 - 04:41:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 183 : backward pass time 12.74 +[Apr 20, 2026 - 04:41:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 183 : optimizer step time 1.04 +[Apr 20, 2026 - 04:41:45 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 183-------------------- +[Apr 20, 2026 - 04:41:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0071 iter count: 1.0 +[Apr 20, 2026 - 04:41:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0071 iter count: 1.0 +[Apr 20, 2026 - 04:41:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 04:41:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 04:42:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121593 43.231056 42.991657 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.889160 69.919922 69.859375 +util 96.250000 99.000000 93.000000 +[Apr 20, 2026 - 04:42:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 184 : avg iteration time 52.51 seconds +[Apr 20, 2026 - 04:42:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 184 : data loading time 0.00 +[Apr 20, 2026 - 04:42:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 184 : avg forward pass time 14.19 +[Apr 20, 2026 - 04:42:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 184 : backward pass time 11.56 +[Apr 20, 2026 - 04:42:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 184 : optimizer step time 0.99 +[Apr 20, 2026 - 04:42:38 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 184-------------------- +[Apr 20, 2026 - 04:42:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0076 iter count: 1.0 +[Apr 20, 2026 - 04:42:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0076 iter count: 1.0 +[Apr 20, 2026 - 04:42:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 04:42:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 04:45:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121245 43.229664 42.991657 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.472656 135.515625 135.402344 +util 71.500000 77.000000 65.000000 +[Apr 20, 2026 - 04:45:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 185 : avg iteration time 186.53 seconds +[Apr 20, 2026 - 04:45:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 185 : data loading time 0.00 +[Apr 20, 2026 - 04:45:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 185 : avg forward pass time 84.07 +[Apr 20, 2026 - 04:45:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 185 : backward pass time 9.77 +[Apr 20, 2026 - 04:45:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 185 : optimizer step time 0.34 +[Apr 20, 2026 - 04:45:44 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 185-------------------- +[Apr 20, 2026 - 04:45:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 38059.8186 iter count: 1.0 +[Apr 20, 2026 - 04:45:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6680 iter count: 1.0 +[Apr 20, 2026 - 04:45:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 38059.8186iter count: 1.0 +[Apr 20, 2026 - 04:45:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 04:45:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.1552iter count: 1.0 +[Apr 20, 2026 - 04:45:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.1412 iter count: 1.0 +[Apr 20, 2026 - 04:45:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.0754 iter count: 1.0 +[Apr 20, 2026 - 04:45:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 75223.8281iter count: 1.0 +[Apr 20, 2026 - 04:45:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 9.8002 iter count: 1.0 +[Apr 20, 2026 - 04:45:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 1398268.6250iter count: 1.0 +[Apr 20, 2026 - 04:45:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6680iter count: 1.0 +[Apr 20, 2026 - 04:45:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 5.2287 iter count: 1.0 +[Apr 20, 2026 - 04:45:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 34800.8633iter count: 1.0 +[Apr 20, 2026 - 04:45:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.4344 iter count: 1.0 +[Apr 20, 2026 - 04:45:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 961.7324iter count: 1.0 +[Apr 20, 2026 - 04:45:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 3.4864 iter count: 1.0 +[Apr 20, 2026 - 04:45:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 1067.2263iter count: 1.0 +[Apr 20, 2026 - 04:45:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 3.6372 iter count: 1.0 +[Apr 20, 2026 - 04:45:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 1442.9303iter count: 1.0 +[Apr 20, 2026 - 04:45:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[Apr 20, 2026 - 04:45:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 04:46:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121315 43.229931 42.991657 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.850098 69.937500 69.744141 +util 94.500000 99.000000 91.000000 +[Apr 20, 2026 - 04:46:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 186 : avg iteration time 56.83 seconds +[Apr 20, 2026 - 04:46:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 186 : data loading time 0.00 +[Apr 20, 2026 - 04:46:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 186 : avg forward pass time 16.37 +[Apr 20, 2026 - 04:46:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 186 : backward pass time 11.54 +[Apr 20, 2026 - 04:46:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 186 : optimizer step time 0.98 +[Apr 20, 2026 - 04:46:41 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 186-------------------- +[Apr 20, 2026 - 04:46:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0076 iter count: 1.0 +[Apr 20, 2026 - 04:46:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0076 iter count: 1.0 +[Apr 20, 2026 - 04:46:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 04:46:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 04:47:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121677 43.231388 42.991657 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.796387 69.839844 69.705078 +util 97.750000 99.000000 94.000000 +[Apr 20, 2026 - 04:47:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 187 : avg iteration time 52.97 seconds +[Apr 20, 2026 - 04:47:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 187 : data loading time 0.00 +[Apr 20, 2026 - 04:47:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 187 : avg forward pass time 13.79 +[Apr 20, 2026 - 04:47:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 187 : backward pass time 11.54 +[Apr 20, 2026 - 04:47:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 187 : optimizer step time 1.01 +[Apr 20, 2026 - 04:47:34 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 187-------------------- +[Apr 20, 2026 - 04:47:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0072 iter count: 1.0 +[Apr 20, 2026 - 04:47:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0072 iter count: 1.0 +[Apr 20, 2026 - 04:47:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 04:47:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 04:48:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121676 43.231388 42.991657 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.864746 69.917969 69.781250 +util 97.000000 99.000000 92.000000 +[Apr 20, 2026 - 04:48:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 188 : avg iteration time 63.71 seconds +[Apr 20, 2026 - 04:48:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 188 : data loading time 0.00 +[Apr 20, 2026 - 04:48:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 188 : avg forward pass time 19.80 +[Apr 20, 2026 - 04:48:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 188 : backward pass time 11.57 +[Apr 20, 2026 - 04:48:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 188 : optimizer step time 0.98 +[Apr 20, 2026 - 04:48:38 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 188-------------------- +[Apr 20, 2026 - 04:48:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0099 iter count: 1.0 +[Apr 20, 2026 - 04:48:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[Apr 20, 2026 - 04:48:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 04:48:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 04:49:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121679 43.231392 42.991657 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.811035 69.898438 69.548828 +util 97.750000 100.000000 93.000000 +[Apr 20, 2026 - 04:49:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 189 : avg iteration time 61.67 seconds +[Apr 20, 2026 - 04:49:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 189 : data loading time 0.00 +[Apr 20, 2026 - 04:49:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 189 : avg forward pass time 18.46 +[Apr 20, 2026 - 04:49:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 189 : backward pass time 11.92 +[Apr 20, 2026 - 04:49:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 189 : optimizer step time 1.28 +[Apr 20, 2026 - 04:49:40 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 189-------------------- +[Apr 20, 2026 - 04:49:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0085 iter count: 1.0 +[Apr 20, 2026 - 04:49:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[Apr 20, 2026 - 04:49:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 04:49:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 04:52:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121462 43.230511 42.991657 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.517578 135.576172 135.437500 +util 71.500000 76.000000 65.000000 +[Apr 20, 2026 - 04:52:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 190 : avg iteration time 174.03 seconds +[Apr 20, 2026 - 04:52:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 190 : data loading time 0.00 +[Apr 20, 2026 - 04:52:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 190 : avg forward pass time 78.19 +[Apr 20, 2026 - 04:52:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 190 : backward pass time 8.69 +[Apr 20, 2026 - 04:52:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 190 : optimizer step time 0.35 +[Apr 20, 2026 - 04:52:34 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 190-------------------- +[Apr 20, 2026 - 04:52:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 62819.8972 iter count: 1.0 +[Apr 20, 2026 - 04:52:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6609 iter count: 1.0 +[Apr 20, 2026 - 04:52:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 62819.8972iter count: 1.0 +[Apr 20, 2026 - 04:52:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 04:52:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.1852iter count: 1.0 +[Apr 20, 2026 - 04:52:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.4827 iter count: 1.0 +[Apr 20, 2026 - 04:52:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.9525 iter count: 1.0 +[Apr 20, 2026 - 04:52:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 97341.6797iter count: 1.0 +[Apr 20, 2026 - 04:52:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 19.4006iter count: 1.0 +[Apr 20, 2026 - 04:52:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 1093538.3750iter count: 1.0 +[Apr 20, 2026 - 04:52:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6609iter count: 1.0 +[Apr 20, 2026 - 04:52:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.9761 iter count: 1.0 +[Apr 20, 2026 - 04:52:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 20998.4863iter count: 1.0 +[Apr 20, 2026 - 04:52:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 2.6462 iter count: 1.0 +[Apr 20, 2026 - 04:52:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 198.8315iter count: 1.0 +[Apr 20, 2026 - 04:52:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 2.8965 iter count: 1.0 +[Apr 20, 2026 - 04:52:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 327.9709iter count: 1.0 +[Apr 20, 2026 - 04:52:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.0105 iter count: 1.0 +[Apr 20, 2026 - 04:52:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 3044.3127iter count: 1.0 +[Apr 20, 2026 - 04:52:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0070 iter count: 1.0 +[Apr 20, 2026 - 04:52:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 04:53:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121462 43.230511 42.991657 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.455566 72.478516 69.781250 +util 95.500000 99.000000 89.000000 +[Apr 20, 2026 - 04:53:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 191 : avg iteration time 51.62 seconds +[Apr 20, 2026 - 04:53:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 191 : data loading time 0.00 +[Apr 20, 2026 - 04:53:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 191 : avg forward pass time 13.42 +[Apr 20, 2026 - 04:53:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 191 : backward pass time 11.84 +[Apr 20, 2026 - 04:53:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 191 : optimizer step time 1.05 +[Apr 20, 2026 - 04:53:25 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 191-------------------- +[Apr 20, 2026 - 04:53:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0086 iter count: 1.0 +[Apr 20, 2026 - 04:53:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[Apr 20, 2026 - 04:53:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 04:53:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 04:54:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121525 43.230762 42.991657 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.489746 72.322266 69.839844 +util 96.250000 100.000000 91.000000 +[Apr 20, 2026 - 04:54:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 192 : avg iteration time 53.12 seconds +[Apr 20, 2026 - 04:54:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 192 : data loading time 0.01 +[Apr 20, 2026 - 04:54:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 192 : avg forward pass time 14.02 +[Apr 20, 2026 - 04:54:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 192 : backward pass time 11.88 +[Apr 20, 2026 - 04:54:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 192 : optimizer step time 1.30 +[Apr 20, 2026 - 04:54:18 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 192-------------------- +[Apr 20, 2026 - 04:54:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0072 iter count: 1.0 +[Apr 20, 2026 - 04:54:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0072 iter count: 1.0 +[Apr 20, 2026 - 04:54:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 04:54:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 04:55:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121528 43.230762 42.991665 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.445801 72.283203 69.820312 +util 97.750000 100.000000 93.000000 +[Apr 20, 2026 - 04:55:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 193 : avg iteration time 53.05 seconds +[Apr 20, 2026 - 04:55:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 193 : data loading time 0.00 +[Apr 20, 2026 - 04:55:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 193 : avg forward pass time 14.05 +[Apr 20, 2026 - 04:55:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 193 : backward pass time 11.94 +[Apr 20, 2026 - 04:55:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 193 : optimizer step time 1.19 +[Apr 20, 2026 - 04:55:11 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 193-------------------- +[Apr 20, 2026 - 04:55:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0071 iter count: 1.0 +[Apr 20, 2026 - 04:55:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0071 iter count: 1.0 +[Apr 20, 2026 - 04:55:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 04:55:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 04:56:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121770 43.231735 42.991661 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.801270 69.937500 69.666016 +util 98.500000 100.000000 94.000000 +[Apr 20, 2026 - 04:56:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 194 : avg iteration time 53.01 seconds +[Apr 20, 2026 - 04:56:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 194 : data loading time 0.00 +[Apr 20, 2026 - 04:56:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 194 : avg forward pass time 13.97 +[Apr 20, 2026 - 04:56:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 194 : backward pass time 11.91 +[Apr 20, 2026 - 04:56:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 194 : optimizer step time 1.19 +[Apr 20, 2026 - 04:56:04 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 194-------------------- +[Apr 20, 2026 - 04:56:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0084 iter count: 1.0 +[Apr 20, 2026 - 04:56:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[Apr 20, 2026 - 04:56:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 04:56:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 04:59:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121528 43.230751 42.991676 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.488770 135.500000 135.478516 +util 68.000000 76.000000 61.000000 +[Apr 20, 2026 - 04:59:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 195 : avg iteration time 185.68 seconds +[Apr 20, 2026 - 04:59:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 195 : data loading time 0.00 +[Apr 20, 2026 - 04:59:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 195 : avg forward pass time 83.99 +[Apr 20, 2026 - 04:59:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 195 : backward pass time 8.74 +[Apr 20, 2026 - 04:59:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 195 : optimizer step time 0.34 +[Apr 20, 2026 - 04:59:10 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 195-------------------- +[Apr 20, 2026 - 04:59:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 288481.8594 iter count: 1.0 +[Apr 20, 2026 - 04:59:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6823 iter count: 1.0 +[Apr 20, 2026 - 04:59:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 288481.8594iter count: 1.0 +[Apr 20, 2026 - 04:59:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 04:59:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.5172iter count: 1.0 +[Apr 20, 2026 - 04:59:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.7590 iter count: 1.0 +[Apr 20, 2026 - 04:59:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.6102 iter count: 1.0 +[Apr 20, 2026 - 04:59:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 377286.2188iter count: 1.0 +[Apr 20, 2026 - 04:59:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 4.5635 iter count: 1.0 +[Apr 20, 2026 - 04:59:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 4074858.0000iter count: 1.0 +[Apr 20, 2026 - 04:59:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6823iter count: 1.0 +[Apr 20, 2026 - 04:59:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.6319 iter count: 1.0 +[Apr 20, 2026 - 04:59:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 1427.7227iter count: 1.0 +[Apr 20, 2026 - 04:59:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 2.8187 iter count: 1.0 +[Apr 20, 2026 - 04:59:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 280.7227iter count: 1.0 +[Apr 20, 2026 - 04:59:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 5.9300 iter count: 1.0 +[Apr 20, 2026 - 04:59:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 141481.5156iter count: 1.0 +[Apr 20, 2026 - 04:59:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 7.1370 iter count: 1.0 +[Apr 20, 2026 - 04:59:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 1581625.8750iter count: 1.0 +[Apr 20, 2026 - 04:59:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[Apr 20, 2026 - 04:59:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 05:00:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121532 43.230728 42.991684 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.815918 69.900391 69.781250 +util 97.000000 100.000000 92.000000 +[Apr 20, 2026 - 05:00:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 196 : avg iteration time 60.09 seconds +[Apr 20, 2026 - 05:00:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 196 : data loading time 0.00 +[Apr 20, 2026 - 05:00:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 196 : avg forward pass time 17.59 +[Apr 20, 2026 - 05:00:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 196 : backward pass time 11.90 +[Apr 20, 2026 - 05:00:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 196 : optimizer step time 1.03 +[Apr 20, 2026 - 05:00:10 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 196-------------------- +[Apr 20, 2026 - 05:00:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0093 iter count: 1.0 +[Apr 20, 2026 - 05:00:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[Apr 20, 2026 - 05:00:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 05:00:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 05:01:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121518 43.230728 42.991661 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.874512 69.919922 69.859375 +util 98.250000 100.000000 95.000000 +[Apr 20, 2026 - 05:01:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 197 : avg iteration time 59.23 seconds +[Apr 20, 2026 - 05:01:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 197 : data loading time 0.00 +[Apr 20, 2026 - 05:01:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 197 : avg forward pass time 16.93 +[Apr 20, 2026 - 05:01:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 197 : backward pass time 11.95 +[Apr 20, 2026 - 05:01:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 197 : optimizer step time 1.12 +[Apr 20, 2026 - 05:01:09 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 197-------------------- +[Apr 20, 2026 - 05:01:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0088 iter count: 1.0 +[Apr 20, 2026 - 05:01:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[Apr 20, 2026 - 05:01:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 05:01:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 05:02:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121530 43.230774 42.991661 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.874512 69.900391 69.839844 +util 98.000000 100.000000 94.000000 +[Apr 20, 2026 - 05:02:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 198 : avg iteration time 58.74 seconds +[Apr 20, 2026 - 05:02:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 198 : data loading time 0.00 +[Apr 20, 2026 - 05:02:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 198 : avg forward pass time 16.84 +[Apr 20, 2026 - 05:02:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 198 : backward pass time 11.92 +[Apr 20, 2026 - 05:02:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 198 : optimizer step time 1.12 +[Apr 20, 2026 - 05:02:08 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 198-------------------- +[Apr 20, 2026 - 05:02:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0112 iter count: 1.0 +[Apr 20, 2026 - 05:02:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[Apr 20, 2026 - 05:02:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 05:02:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 05:03:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121524 43.230728 42.991661 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.752441 69.820312 69.666016 +util 96.500000 99.000000 91.000000 +[Apr 20, 2026 - 05:03:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 199 : avg iteration time 53.10 seconds +[Apr 20, 2026 - 05:03:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 199 : data loading time 0.00 +[Apr 20, 2026 - 05:03:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 199 : avg forward pass time 14.06 +[Apr 20, 2026 - 05:03:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 199 : backward pass time 11.94 +[Apr 20, 2026 - 05:03:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 199 : optimizer step time 1.16 +[Apr 20, 2026 - 05:03:01 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 199-------------------- +[Apr 20, 2026 - 05:03:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0073 iter count: 1.0 +[Apr 20, 2026 - 05:03:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0073 iter count: 1.0 +[Apr 20, 2026 - 05:03:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 05:03:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 05:05:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121230 43.229576 42.991661 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.507812 135.517578 135.496094 +util 67.500000 75.000000 62.000000 +[Apr 20, 2026 - 05:05:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 200 : avg iteration time 168.24 seconds +[Apr 20, 2026 - 05:05:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 200 : data loading time 0.00 +[Apr 20, 2026 - 05:05:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 200 : avg forward pass time 75.28 +[Apr 20, 2026 - 05:05:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 200 : backward pass time 8.70 +[Apr 20, 2026 - 05:05:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 200 : optimizer step time 0.36 +[Apr 20, 2026 - 05:05:49 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 200-------------------- +[Apr 20, 2026 - 05:05:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 10997.7196 iter count: 1.0 +[Apr 20, 2026 - 05:05:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5813 iter count: 1.0 +[Apr 20, 2026 - 05:05:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 10997.7196iter count: 1.0 +[Apr 20, 2026 - 05:05:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 05:05:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.9645iter count: 1.0 +[Apr 20, 2026 - 05:05:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.0095 iter count: 1.0 +[Apr 20, 2026 - 05:05:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 5.9538 iter count: 1.0 +[Apr 20, 2026 - 05:05:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 20207.2109iter count: 1.0 +[Apr 20, 2026 - 05:05:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 7.5311 iter count: 1.0 +[Apr 20, 2026 - 05:05:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 148392.4219iter count: 1.0 +[Apr 20, 2026 - 05:05:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5813iter count: 1.0 +[Apr 20, 2026 - 05:05:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 2.9679 iter count: 1.0 +[Apr 20, 2026 - 05:05:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 378.3659iter count: 1.0 +[Apr 20, 2026 - 05:05:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.2475 iter count: 1.0 +[Apr 20, 2026 - 05:05:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 4890.1294iter count: 1.0 +[Apr 20, 2026 - 05:05:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.4627 iter count: 1.0 +[Apr 20, 2026 - 05:05:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 7519.8862iter count: 1.0 +[Apr 20, 2026 - 05:05:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 2.7218 iter count: 1.0 +[Apr 20, 2026 - 05:05:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 231.2763iter count: 1.0 +[Apr 20, 2026 - 05:05:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0080 iter count: 1.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[Apr 20, 2026 - 05:06:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 20.86 seconds +[Apr 20, 2026 - 05:06:10 | INFO | fastgen.trainer:save_checkpoint:266 ] Saving checkpoint iteration 200 +[Apr 20, 2026 - 05:06:10 | INFO | fastgen.utils.checkpointer:save:301 ] Saving FSDP model to prefix /tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000200 +[Apr 20, 2026 - 05:06:24 | SUCCESS | fastgen.utils.checkpointer:save:329 ] Model saved at iteration 200 +[Apr 20, 2026 - 05:06:24 | INFO | fastgen.trainer:validate:358 ] Validation iteration 200 +[Apr 20, 2026 - 05:06:25 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 05:06:28 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 3.5s +[Apr 20, 2026 - 05:06:32 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 05:06:35 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 2.9s +[Apr 20, 2026 - 05:06:38 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 05:06:41 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 3.0s +[Apr 20, 2026 - 05:06:44 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 05:06:47 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 3.0s +[Apr 20, 2026 - 05:06:50 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 05:06:53 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 2.9s +[Apr 20, 2026 - 05:06:57 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 05:07:00 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 3.0s +[Apr 20, 2026 - 05:07:03 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 05:07:06 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 2.9s +[Apr 20, 2026 - 05:07:09 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 05:07:12 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 2.9s +[Apr 20, 2026 - 05:07:15 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 05:07:18 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 2.9s +[Apr 20, 2026 - 05:07:22 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 05:07:24 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 2.9s +[Apr 20, 2026 - 05:07:28 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging val0 stats at iteration 200-------------------- +[Apr 20, 2026 - 05:07:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0000 iter count: 10.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[Apr 20, 2026 - 05:07:44 | INFO | fastgen.callbacks.wandb:on_validation_end:683 ] Logged 10 val videos at iteration 200 +[Apr 20, 2026 - 05:08:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122164 43.233990 42.991615 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.962402 70.232422 69.820312 +util 97.250000 100.000000 93.000000 +[Apr 20, 2026 - 05:08:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 201 : avg iteration time 170.24 seconds +[Apr 20, 2026 - 05:08:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 201 : data loading time 0.00 +[Apr 20, 2026 - 05:08:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 201 : avg forward pass time 15.10 +[Apr 20, 2026 - 05:08:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 201 : backward pass time 11.91 +[Apr 20, 2026 - 05:08:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 201 : optimizer step time 1.19 +[Apr 20, 2026 - 05:08:40 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 201-------------------- +[Apr 20, 2026 - 05:08:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0093 iter count: 1.0 +[Apr 20, 2026 - 05:08:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[Apr 20, 2026 - 05:08:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 05:08:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 05:09:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121748 43.232315 42.991615 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.020996 70.271484 69.859375 +util 98.750000 100.000000 96.000000 +[Apr 20, 2026 - 05:09:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 202 : avg iteration time 55.63 seconds +[Apr 20, 2026 - 05:09:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 202 : data loading time 0.00 +[Apr 20, 2026 - 05:09:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 202 : avg forward pass time 15.42 +[Apr 20, 2026 - 05:09:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 202 : backward pass time 11.81 +[Apr 20, 2026 - 05:09:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 202 : optimizer step time 1.02 +[Apr 20, 2026 - 05:09:35 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 202-------------------- +[Apr 20, 2026 - 05:09:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0093 iter count: 1.0 +[Apr 20, 2026 - 05:09:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[Apr 20, 2026 - 05:09:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 05:09:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 05:10:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121745 43.232315 42.991615 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.952637 70.193359 69.859375 +util 98.500000 100.000000 96.000000 +[Apr 20, 2026 - 05:10:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 203 : avg iteration time 50.27 seconds +[Apr 20, 2026 - 05:10:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 203 : data loading time 0.00 +[Apr 20, 2026 - 05:10:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 203 : avg forward pass time 12.66 +[Apr 20, 2026 - 05:10:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 203 : backward pass time 11.91 +[Apr 20, 2026 - 05:10:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 203 : optimizer step time 1.09 +[Apr 20, 2026 - 05:10:26 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 203-------------------- +[Apr 20, 2026 - 05:10:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0086 iter count: 1.0 +[Apr 20, 2026 - 05:10:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[Apr 20, 2026 - 05:10:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 05:10:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 05:11:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121253 43.230324 42.991638 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.952637 70.193359 69.859375 +util 96.750000 99.000000 92.000000 +[Apr 20, 2026 - 05:11:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 204 : avg iteration time 64.30 seconds +[Apr 20, 2026 - 05:11:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 204 : data loading time 0.00 +[Apr 20, 2026 - 05:11:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 204 : avg forward pass time 19.73 +[Apr 20, 2026 - 05:11:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 204 : backward pass time 11.89 +[Apr 20, 2026 - 05:11:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 204 : optimizer step time 1.05 +[Apr 20, 2026 - 05:11:30 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 204-------------------- +[Apr 20, 2026 - 05:11:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0107 iter count: 1.0 +[Apr 20, 2026 - 05:11:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[Apr 20, 2026 - 05:11:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 05:11:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 05:14:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121430 43.229683 42.992077 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.507812 135.576172 135.421875 +util 71.000000 77.000000 63.000000 +[Apr 20, 2026 - 05:14:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 205 : avg iteration time 182.30 seconds +[Apr 20, 2026 - 05:14:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 205 : data loading time 0.00 +[Apr 20, 2026 - 05:14:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 205 : avg forward pass time 82.36 +[Apr 20, 2026 - 05:14:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 205 : backward pass time 8.67 +[Apr 20, 2026 - 05:14:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 205 : optimizer step time 0.37 +[Apr 20, 2026 - 05:14:32 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 205-------------------- +[Apr 20, 2026 - 05:14:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 64369.2319 iter count: 1.0 +[Apr 20, 2026 - 05:14:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6975 iter count: 1.0 +[Apr 20, 2026 - 05:14:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 64369.2319iter count: 1.0 +[Apr 20, 2026 - 05:14:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 05:14:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.0305iter count: 1.0 +[Apr 20, 2026 - 05:14:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.5763 iter count: 1.0 +[Apr 20, 2026 - 05:14:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.7420 iter count: 1.0 +[Apr 20, 2026 - 05:14:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 89614.9531iter count: 1.0 +[Apr 20, 2026 - 05:14:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 3.1664 iter count: 1.0 +[Apr 20, 2026 - 05:14:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 717776.4375iter count: 1.0 +[Apr 20, 2026 - 05:14:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6975iter count: 1.0 +[Apr 20, 2026 - 05:14:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 6.7420 iter count: 1.0 +[Apr 20, 2026 - 05:14:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 717776.4375iter count: 1.0 +[Apr 20, 2026 - 05:14:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 2.5972 iter count: 1.0 +[Apr 20, 2026 - 05:14:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 180.2677iter count: 1.0 +[Apr 20, 2026 - 05:14:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 5.1773 iter count: 1.0 +[Apr 20, 2026 - 05:14:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 31399.9277iter count: 1.0 +[Apr 20, 2026 - 05:14:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 5.0748 iter count: 1.0 +[Apr 20, 2026 - 05:14:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 25579.7715iter count: 1.0 +[Apr 20, 2026 - 05:14:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0075 iter count: 1.0 +[Apr 20, 2026 - 05:14:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 05:15:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121493 43.229916 42.992077 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.825684 69.898438 69.646484 +util 98.250000 100.000000 95.000000 +[Apr 20, 2026 - 05:15:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 206 : avg iteration time 48.93 seconds +[Apr 20, 2026 - 05:15:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 206 : data loading time 0.00 +[Apr 20, 2026 - 05:15:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 206 : avg forward pass time 12.07 +[Apr 20, 2026 - 05:15:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 206 : backward pass time 11.86 +[Apr 20, 2026 - 05:15:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 206 : optimizer step time 1.03 +[Apr 20, 2026 - 05:15:21 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 206-------------------- +[Apr 20, 2026 - 05:15:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0107 iter count: 1.0 +[Apr 20, 2026 - 05:15:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[Apr 20, 2026 - 05:15:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 05:15:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 05:16:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121500 43.229939 42.992100 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.830566 69.898438 69.646484 +util 98.750000 100.000000 96.000000 +[Apr 20, 2026 - 05:16:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 207 : avg iteration time 56.21 seconds +[Apr 20, 2026 - 05:16:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 207 : data loading time 0.00 +[Apr 20, 2026 - 05:16:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 207 : avg forward pass time 15.55 +[Apr 20, 2026 - 05:16:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 207 : backward pass time 11.93 +[Apr 20, 2026 - 05:16:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 207 : optimizer step time 1.24 +[Apr 20, 2026 - 05:16:17 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 207-------------------- +[Apr 20, 2026 - 05:16:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0078 iter count: 1.0 +[Apr 20, 2026 - 05:16:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0078 iter count: 1.0 +[Apr 20, 2026 - 05:16:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 05:16:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 05:17:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121496 43.229931 42.992077 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.835449 69.898438 69.685547 +util 98.000000 100.000000 94.000000 +[Apr 20, 2026 - 05:17:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 208 : avg iteration time 61.38 seconds +[Apr 20, 2026 - 05:17:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 208 : data loading time 0.00 +[Apr 20, 2026 - 05:17:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 208 : avg forward pass time 18.25 +[Apr 20, 2026 - 05:17:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 208 : backward pass time 11.89 +[Apr 20, 2026 - 05:17:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 208 : optimizer step time 1.11 +[Apr 20, 2026 - 05:17:19 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 208-------------------- +[Apr 20, 2026 - 05:17:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0096 iter count: 1.0 +[Apr 20, 2026 - 05:17:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[Apr 20, 2026 - 05:17:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 05:17:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 05:18:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121563 43.230167 42.992100 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.811035 69.859375 69.666016 +util 98.250000 100.000000 95.000000 +[Apr 20, 2026 - 05:18:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 209 : avg iteration time 61.20 seconds +[Apr 20, 2026 - 05:18:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 209 : data loading time 0.00 +[Apr 20, 2026 - 05:18:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 209 : avg forward pass time 18.17 +[Apr 20, 2026 - 05:18:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 209 : backward pass time 11.86 +[Apr 20, 2026 - 05:18:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 209 : optimizer step time 1.04 +[Apr 20, 2026 - 05:18:20 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 209-------------------- +[Apr 20, 2026 - 05:18:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0091 iter count: 1.0 +[Apr 20, 2026 - 05:18:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[Apr 20, 2026 - 05:18:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 05:18:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 05:21:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121252 43.228924 42.992100 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.537109 135.556641 135.498047 +util 65.000000 75.000000 55.000000 +[Apr 20, 2026 - 05:21:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 210 : avg iteration time 195.64 seconds +[Apr 20, 2026 - 05:21:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 210 : data loading time 0.00 +[Apr 20, 2026 - 05:21:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 210 : avg forward pass time 88.79 +[Apr 20, 2026 - 05:21:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 210 : backward pass time 9.14 +[Apr 20, 2026 - 05:21:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 210 : optimizer step time 0.34 +[Apr 20, 2026 - 05:21:36 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 210-------------------- +[Apr 20, 2026 - 05:21:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 1496050.3213 iter count: 1.0 +[Apr 20, 2026 - 05:21:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6774 iter count: 1.0 +[Apr 20, 2026 - 05:21:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 1496050.3213iter count: 1.0 +[Apr 20, 2026 - 05:21:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 05:21:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.4878iter count: 1.0 +[Apr 20, 2026 - 05:21:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.6089 iter count: 1.0 +[Apr 20, 2026 - 05:21:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 8.8814 iter count: 1.0 +[Apr 20, 2026 - 05:21:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 1702898.6250iter count: 1.0 +[Apr 20, 2026 - 05:21:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 3.3797 iter count: 1.0 +[Apr 20, 2026 - 05:21:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 51799832.0000iter count: 1.0 +[Apr 20, 2026 - 05:21:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6774iter count: 1.0 +[Apr 20, 2026 - 05:21:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 5.1137 iter count: 1.0 +[Apr 20, 2026 - 05:21:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 27652.8945iter count: 1.0 +[Apr 20, 2026 - 05:21:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.9694 iter count: 1.0 +[Apr 20, 2026 - 05:21:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 20717.6992iter count: 1.0 +[Apr 20, 2026 - 05:21:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.1631 iter count: 1.0 +[Apr 20, 2026 - 05:21:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 4131.0005iter count: 1.0 +[Apr 20, 2026 - 05:21:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 1.8545 iter count: 1.0 +[Apr 20, 2026 - 05:21:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 40.8126 iter count: 1.0 +[Apr 20, 2026 - 05:21:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[Apr 20, 2026 - 05:21:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 05:22:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121306 43.229164 42.992077 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.737793 69.878906 69.451172 +util 96.250000 99.000000 92.000000 +[Apr 20, 2026 - 05:22:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 211 : avg iteration time 56.72 seconds +[Apr 20, 2026 - 05:22:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 211 : data loading time 0.00 +[Apr 20, 2026 - 05:22:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 211 : avg forward pass time 15.93 +[Apr 20, 2026 - 05:22:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 211 : backward pass time 11.92 +[Apr 20, 2026 - 05:22:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 211 : optimizer step time 1.11 +[Apr 20, 2026 - 05:22:32 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 211-------------------- +[Apr 20, 2026 - 05:22:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0112 iter count: 1.0 +[Apr 20, 2026 - 05:22:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[Apr 20, 2026 - 05:22:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 05:22:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 05:23:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121748 43.230885 42.992100 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.679199 69.820312 69.451172 +util 97.750000 100.000000 93.000000 +[Apr 20, 2026 - 05:23:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 212 : avg iteration time 58.22 seconds +[Apr 20, 2026 - 05:23:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 212 : data loading time 0.00 +[Apr 20, 2026 - 05:23:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 212 : avg forward pass time 16.70 +[Apr 20, 2026 - 05:23:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 212 : backward pass time 11.90 +[Apr 20, 2026 - 05:23:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 212 : optimizer step time 1.08 +[Apr 20, 2026 - 05:23:30 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 212-------------------- +[Apr 20, 2026 - 05:23:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0118 iter count: 1.0 +[Apr 20, 2026 - 05:23:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0118 iter count: 1.0 +[Apr 20, 2026 - 05:23:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 05:23:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 05:24:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121813 43.231213 42.992077 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.728027 69.800781 69.626953 +util 98.500000 100.000000 96.000000 +[Apr 20, 2026 - 05:24:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 213 : avg iteration time 60.90 seconds +[Apr 20, 2026 - 05:24:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 213 : data loading time 0.00 +[Apr 20, 2026 - 05:24:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 213 : avg forward pass time 18.12 +[Apr 20, 2026 - 05:24:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 213 : backward pass time 11.80 +[Apr 20, 2026 - 05:24:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 213 : optimizer step time 1.04 +[Apr 20, 2026 - 05:24:31 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 213-------------------- +[Apr 20, 2026 - 05:24:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0098 iter count: 1.0 +[Apr 20, 2026 - 05:24:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[Apr 20, 2026 - 05:24:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 05:24:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 05:25:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121829 43.231216 42.992092 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.669434 69.742188 69.451172 +util 95.750000 100.000000 92.000000 +[Apr 20, 2026 - 05:25:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 214 : avg iteration time 55.83 seconds +[Apr 20, 2026 - 05:25:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 214 : data loading time 0.00 +[Apr 20, 2026 - 05:25:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 214 : avg forward pass time 15.36 +[Apr 20, 2026 - 05:25:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 214 : backward pass time 11.93 +[Apr 20, 2026 - 05:25:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 214 : optimizer step time 1.29 +[Apr 20, 2026 - 05:25:27 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 214-------------------- +[Apr 20, 2026 - 05:25:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0104 iter count: 1.0 +[Apr 20, 2026 - 05:25:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[Apr 20, 2026 - 05:25:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 05:25:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 05:28:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121781 43.230865 42.992191 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.487793 135.554688 135.400391 +util 66.000000 78.000000 49.000000 +[Apr 20, 2026 - 05:28:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 215 : avg iteration time 196.27 seconds +[Apr 20, 2026 - 05:28:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 215 : data loading time 0.00 +[Apr 20, 2026 - 05:28:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 215 : avg forward pass time 89.14 +[Apr 20, 2026 - 05:28:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 215 : backward pass time 9.17 +[Apr 20, 2026 - 05:28:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 215 : optimizer step time 0.33 +[Apr 20, 2026 - 05:28:43 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 215-------------------- +[Apr 20, 2026 - 05:28:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 54177.8547 iter count: 1.0 +[Apr 20, 2026 - 05:28:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6694 iter count: 1.0 +[Apr 20, 2026 - 05:28:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 54177.8547iter count: 1.0 +[Apr 20, 2026 - 05:28:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 05:28:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.3355iter count: 1.0 +[Apr 20, 2026 - 05:28:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.2687 iter count: 1.0 +[Apr 20, 2026 - 05:28:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.1597 iter count: 1.0 +[Apr 20, 2026 - 05:28:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 87986.2344iter count: 1.0 +[Apr 20, 2026 - 05:28:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 12.6462iter count: 1.0 +[Apr 20, 2026 - 05:28:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 1655229.8750iter count: 1.0 +[Apr 20, 2026 - 05:28:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6694iter count: 1.0 +[Apr 20, 2026 - 05:28:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.7140 iter count: 1.0 +[Apr 20, 2026 - 05:28:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 12430.4941iter count: 1.0 +[Apr 20, 2026 - 05:28:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.0008 iter count: 1.0 +[Apr 20, 2026 - 05:28:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 404.0503iter count: 1.0 +[Apr 20, 2026 - 05:28:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 2.4189 iter count: 1.0 +[Apr 20, 2026 - 05:28:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 126.1887iter count: 1.0 +[Apr 20, 2026 - 05:28:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.7316 iter count: 1.0 +[Apr 20, 2026 - 05:28:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 12878.2236iter count: 1.0 +[Apr 20, 2026 - 05:28:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[Apr 20, 2026 - 05:28:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 05:29:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121782 43.230846 42.992191 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.903809 70.152344 69.470703 +util 98.000000 99.000000 95.000000 +[Apr 20, 2026 - 05:29:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 216 : avg iteration time 48.43 seconds +[Apr 20, 2026 - 05:29:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 216 : data loading time 0.00 +[Apr 20, 2026 - 05:29:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 216 : avg forward pass time 11.83 +[Apr 20, 2026 - 05:29:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 216 : backward pass time 11.81 +[Apr 20, 2026 - 05:29:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 216 : optimizer step time 1.06 +[Apr 20, 2026 - 05:29:32 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 216-------------------- +[Apr 20, 2026 - 05:29:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0107 iter count: 1.0 +[Apr 20, 2026 - 05:29:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[Apr 20, 2026 - 05:29:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 05:29:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 05:30:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121849 43.231091 42.992191 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.757324 69.937500 69.412109 +util 96.500000 100.000000 92.000000 +[Apr 20, 2026 - 05:30:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 217 : avg iteration time 55.75 seconds +[Apr 20, 2026 - 05:30:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 217 : data loading time 0.00 +[Apr 20, 2026 - 05:30:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 217 : avg forward pass time 15.41 +[Apr 20, 2026 - 05:30:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 217 : backward pass time 11.95 +[Apr 20, 2026 - 05:30:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 217 : optimizer step time 1.14 +[Apr 20, 2026 - 05:30:28 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 217-------------------- +[Apr 20, 2026 - 05:30:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0107 iter count: 1.0 +[Apr 20, 2026 - 05:30:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[Apr 20, 2026 - 05:30:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 05:30:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 05:31:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122022 43.231823 42.992195 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.796387 69.937500 69.412109 +util 98.000000 100.000000 94.000000 +[Apr 20, 2026 - 05:31:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 218 : avg iteration time 55.69 seconds +[Apr 20, 2026 - 05:31:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 218 : data loading time 0.00 +[Apr 20, 2026 - 05:31:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 218 : avg forward pass time 15.44 +[Apr 20, 2026 - 05:31:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 218 : backward pass time 11.81 +[Apr 20, 2026 - 05:31:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 218 : optimizer step time 1.02 +[Apr 20, 2026 - 05:31:23 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 218-------------------- +[Apr 20, 2026 - 05:31:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0116 iter count: 1.0 +[Apr 20, 2026 - 05:31:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[Apr 20, 2026 - 05:31:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 05:31:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 05:32:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122025 43.231823 42.992191 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.791504 70.035156 69.412109 +util 98.000000 99.000000 95.000000 +[Apr 20, 2026 - 05:32:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 219 : avg iteration time 50.21 seconds +[Apr 20, 2026 - 05:32:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 219 : data loading time 0.00 +[Apr 20, 2026 - 05:32:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 219 : avg forward pass time 12.67 +[Apr 20, 2026 - 05:32:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 219 : backward pass time 11.88 +[Apr 20, 2026 - 05:32:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 219 : optimizer step time 1.06 +[Apr 20, 2026 - 05:32:14 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 219-------------------- +[Apr 20, 2026 - 05:32:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0107 iter count: 1.0 +[Apr 20, 2026 - 05:32:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[Apr 20, 2026 - 05:32:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 05:32:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 05:35:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121403 43.229351 42.992191 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.487793 135.500000 135.476562 +util 72.250000 76.000000 66.000000 +[Apr 20, 2026 - 05:35:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 220 : avg iteration time 179.60 seconds +[Apr 20, 2026 - 05:35:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 220 : data loading time 0.00 +[Apr 20, 2026 - 05:35:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 220 : avg forward pass time 80.99 +[Apr 20, 2026 - 05:35:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 220 : backward pass time 8.71 +[Apr 20, 2026 - 05:35:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 220 : optimizer step time 0.33 +[Apr 20, 2026 - 05:35:13 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 220-------------------- +[Apr 20, 2026 - 05:35:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 190326.6597 iter count: 1.0 +[Apr 20, 2026 - 05:35:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6603 iter count: 1.0 +[Apr 20, 2026 - 05:35:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 190326.6597iter count: 1.0 +[Apr 20, 2026 - 05:35:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 05:35:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.3151iter count: 1.0 +[Apr 20, 2026 - 05:35:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.4339 iter count: 1.0 +[Apr 20, 2026 - 05:35:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.7134 iter count: 1.0 +[Apr 20, 2026 - 05:35:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 255971.5156iter count: 1.0 +[Apr 20, 2026 - 05:35:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 17.5980iter count: 1.0 +[Apr 20, 2026 - 05:35:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 5009098.0000iter count: 1.0 +[Apr 20, 2026 - 05:35:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6603iter count: 1.0 +[Apr 20, 2026 - 05:35:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 2.2991 iter count: 1.0 +[Apr 20, 2026 - 05:35:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 99.3031 iter count: 1.0 +[Apr 20, 2026 - 05:35:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 6.6985 iter count: 1.0 +[Apr 20, 2026 - 05:35:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 658068.5000iter count: 1.0 +[Apr 20, 2026 - 05:35:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 2.7883 iter count: 1.0 +[Apr 20, 2026 - 05:35:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 264.1611iter count: 1.0 +[Apr 20, 2026 - 05:35:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 6.6605 iter count: 1.0 +[Apr 20, 2026 - 05:35:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 609928.6875iter count: 1.0 +[Apr 20, 2026 - 05:35:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0127 iter count: 1.0 +[Apr 20, 2026 - 05:35:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 05:36:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121537 43.229832 42.992191 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.762207 69.820312 69.666016 +util 98.250000 100.000000 94.000000 +[Apr 20, 2026 - 05:36:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 221 : avg iteration time 48.95 seconds +[Apr 20, 2026 - 05:36:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 221 : data loading time 0.00 +[Apr 20, 2026 - 05:36:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 221 : avg forward pass time 12.06 +[Apr 20, 2026 - 05:36:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 221 : backward pass time 11.86 +[Apr 20, 2026 - 05:36:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 221 : optimizer step time 1.05 +[Apr 20, 2026 - 05:36:02 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 221-------------------- +[Apr 20, 2026 - 05:36:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0102 iter count: 1.0 +[Apr 20, 2026 - 05:36:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[Apr 20, 2026 - 05:36:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 05:36:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 05:37:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121767 43.230808 42.992191 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.815918 69.839844 69.783203 +util 97.750000 100.000000 96.000000 +[Apr 20, 2026 - 05:37:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 222 : avg iteration time 59.35 seconds +[Apr 20, 2026 - 05:37:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 222 : data loading time 0.00 +[Apr 20, 2026 - 05:37:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 222 : avg forward pass time 17.09 +[Apr 20, 2026 - 05:37:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 222 : backward pass time 12.04 +[Apr 20, 2026 - 05:37:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 222 : optimizer step time 1.17 +[Apr 20, 2026 - 05:37:01 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 222-------------------- +[Apr 20, 2026 - 05:37:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0126 iter count: 1.0 +[Apr 20, 2026 - 05:37:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0126 iter count: 1.0 +[Apr 20, 2026 - 05:37:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 05:37:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 05:37:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121768 43.230808 42.992195 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.786621 69.839844 69.666016 +util 98.000000 100.000000 94.000000 +[Apr 20, 2026 - 05:37:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 223 : avg iteration time 55.90 seconds +[Apr 20, 2026 - 05:37:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 223 : data loading time 0.00 +[Apr 20, 2026 - 05:37:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 223 : avg forward pass time 15.52 +[Apr 20, 2026 - 05:37:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 223 : backward pass time 11.92 +[Apr 20, 2026 - 05:37:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 223 : optimizer step time 1.04 +[Apr 20, 2026 - 05:37:57 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 223-------------------- +[Apr 20, 2026 - 05:37:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0125 iter count: 1.0 +[Apr 20, 2026 - 05:37:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0125 iter count: 1.0 +[Apr 20, 2026 - 05:37:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 05:37:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 05:38:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121842 43.231060 42.992195 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.820801 69.937500 69.607422 +util 96.750000 100.000000 93.000000 +[Apr 20, 2026 - 05:38:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 224 : avg iteration time 50.67 seconds +[Apr 20, 2026 - 05:38:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 224 : data loading time 0.00 +[Apr 20, 2026 - 05:38:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 224 : avg forward pass time 12.75 +[Apr 20, 2026 - 05:38:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 224 : backward pass time 12.00 +[Apr 20, 2026 - 05:38:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 224 : optimizer step time 1.30 +[Apr 20, 2026 - 05:38:48 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 224-------------------- +[Apr 20, 2026 - 05:38:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0113 iter count: 1.0 +[Apr 20, 2026 - 05:38:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[Apr 20, 2026 - 05:38:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 05:38:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 05:41:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121486 43.229679 42.992195 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.513184 135.556641 135.460938 +util 68.750000 76.000000 64.000000 +[Apr 20, 2026 - 05:41:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 225 : avg iteration time 190.20 seconds +[Apr 20, 2026 - 05:41:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 225 : data loading time 0.00 +[Apr 20, 2026 - 05:41:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 225 : avg forward pass time 86.37 +[Apr 20, 2026 - 05:41:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 225 : backward pass time 8.60 +[Apr 20, 2026 - 05:41:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 225 : optimizer step time 0.33 +[Apr 20, 2026 - 05:41:58 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 225-------------------- +[Apr 20, 2026 - 05:41:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 24138.7871 iter count: 1.0 +[Apr 20, 2026 - 05:41:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6368 iter count: 1.0 +[Apr 20, 2026 - 05:41:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 24138.7871iter count: 1.0 +[Apr 20, 2026 - 05:41:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 05:41:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.1460iter count: 1.0 +[Apr 20, 2026 - 05:41:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.6778 iter count: 1.0 +[Apr 20, 2026 - 05:41:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.5601 iter count: 1.0 +[Apr 20, 2026 - 05:41:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 42191.2695iter count: 1.0 +[Apr 20, 2026 - 05:41:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 3.8787 iter count: 1.0 +[Apr 20, 2026 - 05:41:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 498894.3438iter count: 1.0 +[Apr 20, 2026 - 05:41:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6368iter count: 1.0 +[Apr 20, 2026 - 05:41:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 1.4323 iter count: 1.0 +[Apr 20, 2026 - 05:41:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 17.5426 iter count: 1.0 +[Apr 20, 2026 - 05:41:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.4551 iter count: 1.0 +[Apr 20, 2026 - 05:41:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 7407.0586iter count: 1.0 +[Apr 20, 2026 - 05:41:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.0938 iter count: 1.0 +[Apr 20, 2026 - 05:41:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 3595.8521iter count: 1.0 +[Apr 20, 2026 - 05:41:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.8885 iter count: 1.0 +[Apr 20, 2026 - 05:41:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 17622.0293iter count: 1.0 +[Apr 20, 2026 - 05:41:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[Apr 20, 2026 - 05:41:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 05:42:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121667 43.230404 42.992195 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.820801 69.917969 69.666016 +util 96.250000 99.000000 92.000000 +[Apr 20, 2026 - 05:42:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 226 : avg iteration time 54.44 seconds +[Apr 20, 2026 - 05:42:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 226 : data loading time 0.00 +[Apr 20, 2026 - 05:42:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 226 : avg forward pass time 14.80 +[Apr 20, 2026 - 05:42:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 226 : backward pass time 11.89 +[Apr 20, 2026 - 05:42:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 226 : optimizer step time 1.06 +[Apr 20, 2026 - 05:42:53 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 226-------------------- +[Apr 20, 2026 - 05:42:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0121 iter count: 1.0 +[Apr 20, 2026 - 05:42:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[Apr 20, 2026 - 05:42:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 05:42:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 05:43:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121737 43.230652 42.992195 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.762207 69.839844 69.646484 +util 98.250000 99.000000 96.000000 +[Apr 20, 2026 - 05:43:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 227 : avg iteration time 50.11 seconds +[Apr 20, 2026 - 05:43:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 227 : data loading time 0.00 +[Apr 20, 2026 - 05:43:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 227 : avg forward pass time 12.61 +[Apr 20, 2026 - 05:43:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 227 : backward pass time 11.93 +[Apr 20, 2026 - 05:43:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 227 : optimizer step time 1.12 +[Apr 20, 2026 - 05:43:43 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 227-------------------- +[Apr 20, 2026 - 05:43:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0111 iter count: 1.0 +[Apr 20, 2026 - 05:43:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[Apr 20, 2026 - 05:43:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 05:43:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 05:44:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121790 43.230896 42.992195 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.835449 69.957031 69.626953 +util 97.250000 100.000000 92.000000 +[Apr 20, 2026 - 05:44:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 228 : avg iteration time 58.62 seconds +[Apr 20, 2026 - 05:44:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 228 : data loading time 0.00 +[Apr 20, 2026 - 05:44:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 228 : avg forward pass time 16.82 +[Apr 20, 2026 - 05:44:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 228 : backward pass time 11.89 +[Apr 20, 2026 - 05:44:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 228 : optimizer step time 1.22 +[Apr 20, 2026 - 05:44:41 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 228-------------------- +[Apr 20, 2026 - 05:44:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0095 iter count: 1.0 +[Apr 20, 2026 - 05:44:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[Apr 20, 2026 - 05:44:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 05:44:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 05:45:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121793 43.230892 42.992195 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.815918 69.937500 69.666016 +util 96.750000 100.000000 91.000000 +[Apr 20, 2026 - 05:45:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 229 : avg iteration time 60.93 seconds +[Apr 20, 2026 - 05:45:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 229 : data loading time 0.00 +[Apr 20, 2026 - 05:45:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 229 : avg forward pass time 18.04 +[Apr 20, 2026 - 05:45:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 229 : backward pass time 11.92 +[Apr 20, 2026 - 05:45:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 229 : optimizer step time 1.10 +[Apr 20, 2026 - 05:45:42 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 229-------------------- +[Apr 20, 2026 - 05:45:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0108 iter count: 1.0 +[Apr 20, 2026 - 05:45:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[Apr 20, 2026 - 05:45:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 05:45:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 05:48:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121433 43.229469 42.992195 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.547363 135.597656 135.517578 +util 70.750000 78.000000 66.000000 +[Apr 20, 2026 - 05:48:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 230 : avg iteration time 184.33 seconds +[Apr 20, 2026 - 05:48:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 230 : data loading time 0.00 +[Apr 20, 2026 - 05:48:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 230 : avg forward pass time 83.39 +[Apr 20, 2026 - 05:48:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 230 : backward pass time 8.64 +[Apr 20, 2026 - 05:48:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 230 : optimizer step time 0.33 +[Apr 20, 2026 - 05:48:47 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 230-------------------- +[Apr 20, 2026 - 05:48:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 55966.5569 iter count: 1.0 +[Apr 20, 2026 - 05:48:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6382 iter count: 1.0 +[Apr 20, 2026 - 05:48:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 55966.5569iter count: 1.0 +[Apr 20, 2026 - 05:48:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 05:48:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.1924iter count: 1.0 +[Apr 20, 2026 - 05:48:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.1833 iter count: 1.0 +[Apr 20, 2026 - 05:48:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.3785 iter count: 1.0 +[Apr 20, 2026 - 05:48:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 113672.6484iter count: 1.0 +[Apr 20, 2026 - 05:48:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 10.6611iter count: 1.0 +[Apr 20, 2026 - 05:48:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 2563623.5000iter count: 1.0 +[Apr 20, 2026 - 05:48:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6382iter count: 1.0 +[Apr 20, 2026 - 05:48:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.7264 iter count: 1.0 +[Apr 20, 2026 - 05:48:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 1724.7395iter count: 1.0 +[Apr 20, 2026 - 05:48:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 5.4731 iter count: 1.0 +[Apr 20, 2026 - 05:48:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 56733.2422iter count: 1.0 +[Apr 20, 2026 - 05:48:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 3.2339 iter count: 1.0 +[Apr 20, 2026 - 05:48:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 644.0948iter count: 1.0 +[Apr 20, 2026 - 05:48:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.4905 iter count: 1.0 +[Apr 20, 2026 - 05:48:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 7950.5405iter count: 1.0 +[Apr 20, 2026 - 05:48:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[Apr 20, 2026 - 05:48:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 05:49:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121439 43.229492 42.992195 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 71.192871 72.730469 69.644531 +util 97.250000 100.000000 95.000000 +[Apr 20, 2026 - 05:49:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 231 : avg iteration time 54.40 seconds +[Apr 20, 2026 - 05:49:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 231 : data loading time 0.00 +[Apr 20, 2026 - 05:49:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 231 : avg forward pass time 14.76 +[Apr 20, 2026 - 05:49:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 231 : backward pass time 11.86 +[Apr 20, 2026 - 05:49:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 231 : optimizer step time 1.17 +[Apr 20, 2026 - 05:49:41 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 231-------------------- +[Apr 20, 2026 - 05:49:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0092 iter count: 1.0 +[Apr 20, 2026 - 05:49:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[Apr 20, 2026 - 05:49:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 05:49:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 05:50:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121748 43.230713 42.992210 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.708496 69.742188 69.666016 +util 97.000000 100.000000 94.000000 +[Apr 20, 2026 - 05:50:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 232 : avg iteration time 58.59 seconds +[Apr 20, 2026 - 05:50:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 232 : data loading time 0.00 +[Apr 20, 2026 - 05:50:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 232 : avg forward pass time 16.79 +[Apr 20, 2026 - 05:50:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 232 : backward pass time 11.87 +[Apr 20, 2026 - 05:50:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 232 : optimizer step time 1.20 +[Apr 20, 2026 - 05:50:40 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 232-------------------- +[Apr 20, 2026 - 05:50:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0137 iter count: 1.0 +[Apr 20, 2026 - 05:50:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0137 iter count: 1.0 +[Apr 20, 2026 - 05:50:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 05:50:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 05:51:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121808 43.230965 42.992195 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.713379 69.878906 69.468750 +util 96.500000 99.000000 94.000000 +[Apr 20, 2026 - 05:51:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 233 : avg iteration time 52.61 seconds +[Apr 20, 2026 - 05:51:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 233 : data loading time 0.00 +[Apr 20, 2026 - 05:51:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 233 : avg forward pass time 13.93 +[Apr 20, 2026 - 05:51:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 233 : backward pass time 11.87 +[Apr 20, 2026 - 05:51:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 233 : optimizer step time 1.06 +[Apr 20, 2026 - 05:51:32 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 233-------------------- +[Apr 20, 2026 - 05:51:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0075 iter count: 1.0 +[Apr 20, 2026 - 05:51:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0075 iter count: 1.0 +[Apr 20, 2026 - 05:51:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 05:51:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 05:52:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121767 43.230801 42.992195 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 71.139160 72.710938 69.449219 +util 97.500000 100.000000 93.000000 +[Apr 20, 2026 - 05:52:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 234 : avg iteration time 52.70 seconds +[Apr 20, 2026 - 05:52:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 234 : data loading time 0.00 +[Apr 20, 2026 - 05:52:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 234 : avg forward pass time 13.99 +[Apr 20, 2026 - 05:52:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 234 : backward pass time 11.79 +[Apr 20, 2026 - 05:52:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 234 : optimizer step time 1.03 +[Apr 20, 2026 - 05:52:25 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 234-------------------- +[Apr 20, 2026 - 05:52:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0071 iter count: 1.0 +[Apr 20, 2026 - 05:52:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0071 iter count: 1.0 +[Apr 20, 2026 - 05:52:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 05:52:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 05:55:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121637 43.230286 42.992195 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.512207 135.519531 135.498047 +util 72.250000 76.000000 66.000000 +[Apr 20, 2026 - 05:55:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 235 : avg iteration time 187.24 seconds +[Apr 20, 2026 - 05:55:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 235 : data loading time 0.00 +[Apr 20, 2026 - 05:55:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 235 : avg forward pass time 84.86 +[Apr 20, 2026 - 05:55:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 235 : backward pass time 8.65 +[Apr 20, 2026 - 05:55:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 235 : optimizer step time 0.34 +[Apr 20, 2026 - 05:55:32 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 235-------------------- +[Apr 20, 2026 - 05:55:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 591906.6621 iter count: 1.0 +[Apr 20, 2026 - 05:55:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6231 iter count: 1.0 +[Apr 20, 2026 - 05:55:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 591906.6621iter count: 1.0 +[Apr 20, 2026 - 05:55:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 05:55:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.6457iter count: 1.0 +[Apr 20, 2026 - 05:55:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.7002 iter count: 1.0 +[Apr 20, 2026 - 05:55:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 8.3000 iter count: 1.0 +[Apr 20, 2026 - 05:55:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 845437.8750iter count: 1.0 +[Apr 20, 2026 - 05:55:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 4.0565 iter count: 1.0 +[Apr 20, 2026 - 05:55:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 16191740.0000iter count: 1.0 +[Apr 20, 2026 - 05:55:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6231iter count: 1.0 +[Apr 20, 2026 - 05:55:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 5.3238 iter count: 1.0 +[Apr 20, 2026 - 05:55:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 42089.1328iter count: 1.0 +[Apr 20, 2026 - 05:55:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.0449 iter count: 1.0 +[Apr 20, 2026 - 05:55:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 441.3112iter count: 1.0 +[Apr 20, 2026 - 05:55:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.1827 iter count: 1.0 +[Apr 20, 2026 - 05:55:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 4296.0029iter count: 1.0 +[Apr 20, 2026 - 05:55:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 2.5048 iter count: 1.0 +[Apr 20, 2026 - 05:55:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 149.8407iter count: 1.0 +[Apr 20, 2026 - 05:55:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[Apr 20, 2026 - 05:55:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 05:56:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121766 43.230782 42.992214 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.826660 69.978516 69.607422 +util 98.250000 99.000000 96.000000 +[Apr 20, 2026 - 05:56:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 236 : avg iteration time 65.60 seconds +[Apr 20, 2026 - 05:56:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 236 : data loading time 0.00 +[Apr 20, 2026 - 05:56:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 236 : avg forward pass time 20.41 +[Apr 20, 2026 - 05:56:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 236 : backward pass time 11.88 +[Apr 20, 2026 - 05:56:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 236 : optimizer step time 1.05 +[Apr 20, 2026 - 05:56:38 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 236-------------------- +[Apr 20, 2026 - 05:56:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0108 iter count: 1.0 +[Apr 20, 2026 - 05:56:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[Apr 20, 2026 - 05:56:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 05:56:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 05:57:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121768 43.230804 42.992195 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.777832 69.919922 69.568359 +util 98.500000 100.000000 95.000000 +[Apr 20, 2026 - 05:57:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 237 : avg iteration time 61.13 seconds +[Apr 20, 2026 - 05:57:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 237 : data loading time 0.00 +[Apr 20, 2026 - 05:57:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 237 : avg forward pass time 18.17 +[Apr 20, 2026 - 05:57:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 237 : backward pass time 11.79 +[Apr 20, 2026 - 05:57:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 237 : optimizer step time 1.03 +[Apr 20, 2026 - 05:57:39 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 237-------------------- +[Apr 20, 2026 - 05:57:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0106 iter count: 1.0 +[Apr 20, 2026 - 05:57:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[Apr 20, 2026 - 05:57:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 05:57:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 05:58:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121761 43.230782 42.992195 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.339355 71.931641 69.781250 +util 97.500000 99.000000 93.000000 +[Apr 20, 2026 - 05:58:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 238 : avg iteration time 53.04 seconds +[Apr 20, 2026 - 05:58:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 238 : data loading time 0.00 +[Apr 20, 2026 - 05:58:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 238 : avg forward pass time 14.11 +[Apr 20, 2026 - 05:58:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 238 : backward pass time 11.81 +[Apr 20, 2026 - 05:58:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 238 : optimizer step time 1.04 +[Apr 20, 2026 - 05:58:32 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 238-------------------- +[Apr 20, 2026 - 05:58:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0094 iter count: 1.0 +[Apr 20, 2026 - 05:58:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[Apr 20, 2026 - 05:58:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 05:58:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 05:59:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121762 43.230785 42.992195 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.349121 71.931641 69.781250 +util 98.250000 99.000000 96.000000 +[Apr 20, 2026 - 05:59:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 239 : avg iteration time 47.43 seconds +[Apr 20, 2026 - 05:59:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 239 : data loading time 0.00 +[Apr 20, 2026 - 05:59:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 239 : avg forward pass time 11.27 +[Apr 20, 2026 - 05:59:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 239 : backward pass time 11.95 +[Apr 20, 2026 - 05:59:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 239 : optimizer step time 1.12 +[Apr 20, 2026 - 05:59:19 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 239-------------------- +[Apr 20, 2026 - 05:59:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0096 iter count: 1.0 +[Apr 20, 2026 - 05:59:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[Apr 20, 2026 - 05:59:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 05:59:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 06:02:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121438 43.229488 42.992195 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.458496 135.517578 135.402344 +util 69.000000 76.000000 65.000000 +[Apr 20, 2026 - 06:02:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 240 : avg iteration time 189.79 seconds +[Apr 20, 2026 - 06:02:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 240 : data loading time 0.00 +[Apr 20, 2026 - 06:02:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 240 : avg forward pass time 86.05 +[Apr 20, 2026 - 06:02:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 240 : backward pass time 8.70 +[Apr 20, 2026 - 06:02:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 240 : optimizer step time 0.42 +[Apr 20, 2026 - 06:02:29 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 240-------------------- +[Apr 20, 2026 - 06:02:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 119200.8955 iter count: 1.0 +[Apr 20, 2026 - 06:02:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6753 iter count: 1.0 +[Apr 20, 2026 - 06:02:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 119200.8955iter count: 1.0 +[Apr 20, 2026 - 06:02:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 06:02:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.0576iter count: 1.0 +[Apr 20, 2026 - 06:02:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.3154 iter count: 1.0 +[Apr 20, 2026 - 06:02:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.6306 iter count: 1.0 +[Apr 20, 2026 - 06:02:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 192123.9219iter count: 1.0 +[Apr 20, 2026 - 06:02:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 13.8850iter count: 1.0 +[Apr 20, 2026 - 06:02:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 4244478.0000iter count: 1.0 +[Apr 20, 2026 - 06:02:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6753iter count: 1.0 +[Apr 20, 2026 - 06:02:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.7026 iter count: 1.0 +[Apr 20, 2026 - 06:02:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 12152.0645iter count: 1.0 +[Apr 20, 2026 - 06:02:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.4364 iter count: 1.0 +[Apr 20, 2026 - 06:02:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 965.6495iter count: 1.0 +[Apr 20, 2026 - 06:02:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 5.0899 iter count: 1.0 +[Apr 20, 2026 - 06:02:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 26367.5586iter count: 1.0 +[Apr 20, 2026 - 06:02:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.3537 iter count: 1.0 +[Apr 20, 2026 - 06:02:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 6047.6792iter count: 1.0 +[Apr 20, 2026 - 06:02:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[Apr 20, 2026 - 06:02:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 06:03:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121494 43.229713 42.992195 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.894043 69.978516 69.683594 +util 97.000000 100.000000 93.000000 +[Apr 20, 2026 - 06:03:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 241 : avg iteration time 60.13 seconds +[Apr 20, 2026 - 06:03:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 241 : data loading time 0.00 +[Apr 20, 2026 - 06:03:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 241 : avg forward pass time 17.53 +[Apr 20, 2026 - 06:03:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 241 : backward pass time 11.88 +[Apr 20, 2026 - 06:03:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 241 : optimizer step time 1.29 +[Apr 20, 2026 - 06:03:29 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 241-------------------- +[Apr 20, 2026 - 06:03:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0093 iter count: 1.0 +[Apr 20, 2026 - 06:03:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[Apr 20, 2026 - 06:03:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 06:03:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 06:04:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121872 43.231205 42.992199 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.835449 70.015625 69.626953 +util 97.500000 100.000000 95.000000 +[Apr 20, 2026 - 06:04:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 242 : avg iteration time 50.16 seconds +[Apr 20, 2026 - 06:04:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 242 : data loading time 0.00 +[Apr 20, 2026 - 06:04:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 242 : avg forward pass time 12.69 +[Apr 20, 2026 - 06:04:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 242 : backward pass time 11.82 +[Apr 20, 2026 - 06:04:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 242 : optimizer step time 1.06 +[Apr 20, 2026 - 06:04:19 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 242-------------------- +[Apr 20, 2026 - 06:04:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0100 iter count: 1.0 +[Apr 20, 2026 - 06:04:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[Apr 20, 2026 - 06:04:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 06:04:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 06:05:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121867 43.231205 42.992195 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.796387 69.917969 69.607422 +util 98.750000 100.000000 97.000000 +[Apr 20, 2026 - 06:05:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 243 : avg iteration time 50.13 seconds +[Apr 20, 2026 - 06:05:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 243 : data loading time 0.00 +[Apr 20, 2026 - 06:05:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 243 : avg forward pass time 12.74 +[Apr 20, 2026 - 06:05:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 243 : backward pass time 11.80 +[Apr 20, 2026 - 06:05:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 243 : optimizer step time 1.01 +[Apr 20, 2026 - 06:05:10 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 243-------------------- +[Apr 20, 2026 - 06:05:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0094 iter count: 1.0 +[Apr 20, 2026 - 06:05:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[Apr 20, 2026 - 06:05:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 06:05:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 06:06:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121930 43.231457 42.992195 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.811035 69.917969 69.666016 +util 94.750000 99.000000 91.000000 +[Apr 20, 2026 - 06:06:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 244 : avg iteration time 64.22 seconds +[Apr 20, 2026 - 06:06:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 244 : data loading time 0.00 +[Apr 20, 2026 - 06:06:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 244 : avg forward pass time 19.60 +[Apr 20, 2026 - 06:06:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 244 : backward pass time 11.94 +[Apr 20, 2026 - 06:06:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 244 : optimizer step time 1.18 +[Apr 20, 2026 - 06:06:14 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 244-------------------- +[Apr 20, 2026 - 06:06:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0110 iter count: 1.0 +[Apr 20, 2026 - 06:06:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[Apr 20, 2026 - 06:06:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 06:06:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 06:09:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121747 43.230724 42.992195 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.526367 135.556641 135.439453 +util 71.000000 75.000000 65.000000 +[Apr 20, 2026 - 06:09:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 245 : avg iteration time 179.54 seconds +[Apr 20, 2026 - 06:09:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 245 : data loading time 0.00 +[Apr 20, 2026 - 06:09:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 245 : avg forward pass time 80.83 +[Apr 20, 2026 - 06:09:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 245 : backward pass time 8.94 +[Apr 20, 2026 - 06:09:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 245 : optimizer step time 0.33 +[Apr 20, 2026 - 06:09:13 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 245-------------------- +[Apr 20, 2026 - 06:09:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 60419.2385 iter count: 1.0 +[Apr 20, 2026 - 06:09:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6505 iter count: 1.0 +[Apr 20, 2026 - 06:09:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 60419.2385iter count: 1.0 +[Apr 20, 2026 - 06:09:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 06:09:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.1851iter count: 1.0 +[Apr 20, 2026 - 06:09:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.1833 iter count: 1.0 +[Apr 20, 2026 - 06:09:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.1378 iter count: 1.0 +[Apr 20, 2026 - 06:09:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 78872.5312iter count: 1.0 +[Apr 20, 2026 - 06:09:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 10.6602iter count: 1.0 +[Apr 20, 2026 - 06:09:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 1584334.2500iter count: 1.0 +[Apr 20, 2026 - 06:09:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6505iter count: 1.0 +[Apr 20, 2026 - 06:09:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 2.8627 iter count: 1.0 +[Apr 20, 2026 - 06:09:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 306.5550iter count: 1.0 +[Apr 20, 2026 - 06:09:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.4020 iter count: 1.0 +[Apr 20, 2026 - 06:09:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 6660.8149iter count: 1.0 +[Apr 20, 2026 - 06:09:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 7.1378 iter count: 1.0 +[Apr 20, 2026 - 06:09:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 1584334.2500iter count: 1.0 +[Apr 20, 2026 - 06:09:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 3.0489 iter count: 1.0 +[Apr 20, 2026 - 06:09:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 444.8344iter count: 1.0 +[Apr 20, 2026 - 06:09:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[Apr 20, 2026 - 06:09:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 06:10:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121928 43.231449 42.992195 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.791504 69.917969 69.607422 +util 97.500000 100.000000 93.000000 +[Apr 20, 2026 - 06:10:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 246 : avg iteration time 60.39 seconds +[Apr 20, 2026 - 06:10:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 246 : data loading time 0.00 +[Apr 20, 2026 - 06:10:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 246 : avg forward pass time 17.80 +[Apr 20, 2026 - 06:10:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 246 : backward pass time 11.83 +[Apr 20, 2026 - 06:10:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 246 : optimizer step time 1.05 +[Apr 20, 2026 - 06:10:14 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 246-------------------- +[Apr 20, 2026 - 06:10:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0135 iter count: 1.0 +[Apr 20, 2026 - 06:10:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0135 iter count: 1.0 +[Apr 20, 2026 - 06:10:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 06:10:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 06:11:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122051 43.231941 42.992195 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.732910 69.898438 69.470703 +util 95.750000 99.000000 91.000000 +[Apr 20, 2026 - 06:11:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 247 : avg iteration time 53.32 seconds +[Apr 20, 2026 - 06:11:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 247 : data loading time 0.00 +[Apr 20, 2026 - 06:11:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 247 : avg forward pass time 14.23 +[Apr 20, 2026 - 06:11:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 247 : backward pass time 11.90 +[Apr 20, 2026 - 06:11:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 247 : optimizer step time 1.07 +[Apr 20, 2026 - 06:11:07 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 247-------------------- +[Apr 20, 2026 - 06:11:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0110 iter count: 1.0 +[Apr 20, 2026 - 06:11:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[Apr 20, 2026 - 06:11:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 06:11:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 06:11:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122063 43.231964 42.992195 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.771973 69.917969 69.490234 +util 98.000000 100.000000 94.000000 +[Apr 20, 2026 - 06:11:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 248 : avg iteration time 47.69 seconds +[Apr 20, 2026 - 06:11:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 248 : data loading time 0.00 +[Apr 20, 2026 - 06:11:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 248 : avg forward pass time 11.43 +[Apr 20, 2026 - 06:11:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 248 : backward pass time 11.87 +[Apr 20, 2026 - 06:11:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 248 : optimizer step time 1.03 +[Apr 20, 2026 - 06:11:55 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 248-------------------- +[Apr 20, 2026 - 06:11:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0100 iter count: 1.0 +[Apr 20, 2026 - 06:11:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[Apr 20, 2026 - 06:11:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 06:11:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 06:12:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122062 43.231964 42.992214 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.630371 69.761719 69.353516 +util 95.750000 99.000000 92.000000 +[Apr 20, 2026 - 06:12:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 249 : avg iteration time 53.26 seconds +[Apr 20, 2026 - 06:12:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 249 : data loading time 0.00 +[Apr 20, 2026 - 06:12:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 249 : avg forward pass time 14.09 +[Apr 20, 2026 - 06:12:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 249 : backward pass time 11.95 +[Apr 20, 2026 - 06:12:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 249 : optimizer step time 1.12 +[Apr 20, 2026 - 06:12:48 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 249-------------------- +[Apr 20, 2026 - 06:12:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0113 iter count: 1.0 +[Apr 20, 2026 - 06:12:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[Apr 20, 2026 - 06:12:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 06:12:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 06:15:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121840 43.231049 42.992218 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.526855 135.554688 135.460938 +util 72.000000 80.000000 63.000000 +[Apr 20, 2026 - 06:15:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 250 : avg iteration time 182.40 seconds +[Apr 20, 2026 - 06:15:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 250 : data loading time 0.00 +[Apr 20, 2026 - 06:15:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 250 : avg forward pass time 82.37 +[Apr 20, 2026 - 06:15:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 250 : backward pass time 8.72 +[Apr 20, 2026 - 06:15:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 250 : optimizer step time 0.33 +[Apr 20, 2026 - 06:15:50 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 250-------------------- +[Apr 20, 2026 - 06:15:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 70619.2418 iter count: 1.0 +[Apr 20, 2026 - 06:15:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6068 iter count: 1.0 +[Apr 20, 2026 - 06:15:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 70619.2418iter count: 1.0 +[Apr 20, 2026 - 06:15:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 06:15:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.0902iter count: 1.0 +[Apr 20, 2026 - 06:15:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.0860 iter count: 1.0 +[Apr 20, 2026 - 06:15:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.0251 iter count: 1.0 +[Apr 20, 2026 - 06:15:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 95613.3906iter count: 1.0 +[Apr 20, 2026 - 06:15:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 8.7761 iter count: 1.0 +[Apr 20, 2026 - 06:15:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 1264519.0000iter count: 1.0 +[Apr 20, 2026 - 06:15:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6068iter count: 1.0 +[Apr 20, 2026 - 06:15:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.5345 iter count: 1.0 +[Apr 20, 2026 - 06:15:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 1175.0691iter count: 1.0 +[Apr 20, 2026 - 06:15:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.3936 iter count: 1.0 +[Apr 20, 2026 - 06:15:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 886.3765iter count: 1.0 +[Apr 20, 2026 - 06:15:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 3.1941 iter count: 1.0 +[Apr 20, 2026 - 06:15:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 594.8263iter count: 1.0 +[Apr 20, 2026 - 06:15:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.1983 iter count: 1.0 +[Apr 20, 2026 - 06:15:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 4432.0854iter count: 1.0 +[Apr 20, 2026 - 06:15:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[Apr 20, 2026 - 06:15:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 06:16:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121830 43.231026 42.992210 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.918457 70.015625 69.820312 +util 96.750000 100.000000 92.000000 +[Apr 20, 2026 - 06:16:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 251 : avg iteration time 57.90 seconds +[Apr 20, 2026 - 06:16:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 251 : data loading time 0.00 +[Apr 20, 2026 - 06:16:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 251 : avg forward pass time 16.31 +[Apr 20, 2026 - 06:16:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 251 : backward pass time 12.12 +[Apr 20, 2026 - 06:16:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 251 : optimizer step time 1.10 +[Apr 20, 2026 - 06:16:48 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 251-------------------- +[Apr 20, 2026 - 06:16:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0114 iter count: 1.0 +[Apr 20, 2026 - 06:16:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[Apr 20, 2026 - 06:16:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 06:16:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 06:17:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121825 43.231026 42.992195 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.850098 69.976562 69.626953 +util 96.000000 99.000000 92.000000 +[Apr 20, 2026 - 06:17:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 252 : avg iteration time 52.95 seconds +[Apr 20, 2026 - 06:17:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 252 : data loading time 0.00 +[Apr 20, 2026 - 06:17:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 252 : avg forward pass time 14.10 +[Apr 20, 2026 - 06:17:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 252 : backward pass time 11.84 +[Apr 20, 2026 - 06:17:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 252 : optimizer step time 1.04 +[Apr 20, 2026 - 06:17:41 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 252-------------------- +[Apr 20, 2026 - 06:17:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0094 iter count: 1.0 +[Apr 20, 2026 - 06:17:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[Apr 20, 2026 - 06:17:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 06:17:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 06:18:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121826 43.231041 42.992195 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.962402 70.015625 69.802734 +util 95.250000 99.000000 91.000000 +[Apr 20, 2026 - 06:18:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 253 : avg iteration time 61.18 seconds +[Apr 20, 2026 - 06:18:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 253 : data loading time 0.00 +[Apr 20, 2026 - 06:18:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 253 : avg forward pass time 18.17 +[Apr 20, 2026 - 06:18:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 253 : backward pass time 11.82 +[Apr 20, 2026 - 06:18:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 253 : optimizer step time 1.09 +[Apr 20, 2026 - 06:18:42 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 253-------------------- +[Apr 20, 2026 - 06:18:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0112 iter count: 1.0 +[Apr 20, 2026 - 06:18:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[Apr 20, 2026 - 06:18:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 06:18:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 06:19:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121841 43.231049 42.992218 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.874512 70.035156 69.626953 +util 97.250000 99.000000 95.000000 +[Apr 20, 2026 - 06:19:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 254 : avg iteration time 47.41 seconds +[Apr 20, 2026 - 06:19:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 254 : data loading time 0.00 +[Apr 20, 2026 - 06:19:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 254 : avg forward pass time 11.28 +[Apr 20, 2026 - 06:19:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 254 : backward pass time 11.93 +[Apr 20, 2026 - 06:19:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 254 : optimizer step time 1.09 +[Apr 20, 2026 - 06:19:30 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 254-------------------- +[Apr 20, 2026 - 06:19:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0106 iter count: 1.0 +[Apr 20, 2026 - 06:19:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[Apr 20, 2026 - 06:19:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 06:19:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 06:22:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121490 43.229679 42.992195 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.479004 135.517578 135.402344 +util 71.000000 78.000000 66.000000 +[Apr 20, 2026 - 06:22:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 255 : avg iteration time 179.23 seconds +[Apr 20, 2026 - 06:22:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 255 : data loading time 0.00 +[Apr 20, 2026 - 06:22:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 255 : avg forward pass time 80.84 +[Apr 20, 2026 - 06:22:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 255 : backward pass time 8.63 +[Apr 20, 2026 - 06:22:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 255 : optimizer step time 0.36 +[Apr 20, 2026 - 06:22:29 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 255-------------------- +[Apr 20, 2026 - 06:22:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 32166.9116 iter count: 1.0 +[Apr 20, 2026 - 06:22:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5718 iter count: 1.0 +[Apr 20, 2026 - 06:22:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 32166.9116iter count: 1.0 +[Apr 20, 2026 - 06:22:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 06:22:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.0568iter count: 1.0 +[Apr 20, 2026 - 06:22:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.2228 iter count: 1.0 +[Apr 20, 2026 - 06:22:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.8730 iter count: 1.0 +[Apr 20, 2026 - 06:22:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 81481.5312iter count: 1.0 +[Apr 20, 2026 - 06:22:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 11.5383iter count: 1.0 +[Apr 20, 2026 - 06:22:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 932836.3125iter count: 1.0 +[Apr 20, 2026 - 06:22:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5718iter count: 1.0 +[Apr 20, 2026 - 06:22:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 6.7292 iter count: 1.0 +[Apr 20, 2026 - 06:22:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 699667.6250iter count: 1.0 +[Apr 20, 2026 - 06:22:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.4237 iter count: 1.0 +[Apr 20, 2026 - 06:22:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 941.3732iter count: 1.0 +[Apr 20, 2026 - 06:22:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 3.1374 iter count: 1.0 +[Apr 20, 2026 - 06:22:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 531.0137iter count: 1.0 +[Apr 20, 2026 - 06:22:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 3.9235 iter count: 1.0 +[Apr 20, 2026 - 06:22:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 2557.8635iter count: 1.0 +[Apr 20, 2026 - 06:22:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[Apr 20, 2026 - 06:22:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 06:23:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121547 43.229923 42.992195 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.903809 69.978516 69.839844 +util 97.000000 100.000000 94.000000 +[Apr 20, 2026 - 06:23:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 256 : avg iteration time 62.77 seconds +[Apr 20, 2026 - 06:23:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 256 : data loading time 0.00 +[Apr 20, 2026 - 06:23:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 256 : avg forward pass time 18.99 +[Apr 20, 2026 - 06:23:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 256 : backward pass time 11.89 +[Apr 20, 2026 - 06:23:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 256 : optimizer step time 1.06 +[Apr 20, 2026 - 06:23:32 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 256-------------------- +[Apr 20, 2026 - 06:23:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0150 iter count: 1.0 +[Apr 20, 2026 - 06:23:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0150 iter count: 1.0 +[Apr 20, 2026 - 06:23:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 06:23:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 06:24:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121802 43.230923 42.992195 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.767090 69.917969 69.431641 +util 98.250000 100.000000 95.000000 +[Apr 20, 2026 - 06:24:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 257 : avg iteration time 50.47 seconds +[Apr 20, 2026 - 06:24:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 257 : data loading time 0.00 +[Apr 20, 2026 - 06:24:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 257 : avg forward pass time 12.81 +[Apr 20, 2026 - 06:24:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 257 : backward pass time 11.87 +[Apr 20, 2026 - 06:24:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 257 : optimizer step time 1.03 +[Apr 20, 2026 - 06:24:22 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 257-------------------- +[Apr 20, 2026 - 06:24:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0113 iter count: 1.0 +[Apr 20, 2026 - 06:24:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[Apr 20, 2026 - 06:24:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 06:24:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 06:25:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121869 43.231171 42.992218 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.728027 69.898438 69.412109 +util 98.500000 100.000000 96.000000 +[Apr 20, 2026 - 06:25:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 258 : avg iteration time 58.87 seconds +[Apr 20, 2026 - 06:25:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 258 : data loading time 0.00 +[Apr 20, 2026 - 06:25:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 258 : avg forward pass time 16.93 +[Apr 20, 2026 - 06:25:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 258 : backward pass time 11.99 +[Apr 20, 2026 - 06:25:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 258 : optimizer step time 1.12 +[Apr 20, 2026 - 06:25:21 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 258-------------------- +[Apr 20, 2026 - 06:25:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0114 iter count: 1.0 +[Apr 20, 2026 - 06:25:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[Apr 20, 2026 - 06:25:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 06:25:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 06:26:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121866 43.231174 42.992195 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.811035 69.976562 69.431641 +util 97.500000 100.000000 93.000000 +[Apr 20, 2026 - 06:26:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 259 : avg iteration time 47.51 seconds +[Apr 20, 2026 - 06:26:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 259 : data loading time 0.00 +[Apr 20, 2026 - 06:26:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 259 : avg forward pass time 11.37 +[Apr 20, 2026 - 06:26:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 259 : backward pass time 11.83 +[Apr 20, 2026 - 06:26:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 259 : optimizer step time 1.06 +[Apr 20, 2026 - 06:26:09 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 259-------------------- +[Apr 20, 2026 - 06:26:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0122 iter count: 1.0 +[Apr 20, 2026 - 06:26:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0122 iter count: 1.0 +[Apr 20, 2026 - 06:26:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 06:26:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 06:29:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121603 43.230122 42.992195 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.487793 135.519531 135.457031 +util 70.000000 77.000000 64.000000 +[Apr 20, 2026 - 06:29:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 260 : avg iteration time 188.02 seconds +[Apr 20, 2026 - 06:29:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 260 : data loading time 0.00 +[Apr 20, 2026 - 06:29:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 260 : avg forward pass time 85.17 +[Apr 20, 2026 - 06:29:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 260 : backward pass time 8.70 +[Apr 20, 2026 - 06:29:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 260 : optimizer step time 0.33 +[Apr 20, 2026 - 06:29:17 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 260-------------------- +[Apr 20, 2026 - 06:29:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 139294.4092 iter count: 1.0 +[Apr 20, 2026 - 06:29:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6355 iter count: 1.0 +[Apr 20, 2026 - 06:29:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 139294.4092iter count: 1.0 +[Apr 20, 2026 - 06:29:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 06:29:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.1379iter count: 1.0 +[Apr 20, 2026 - 06:29:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.8161 iter count: 1.0 +[Apr 20, 2026 - 06:29:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.3544 iter count: 1.0 +[Apr 20, 2026 - 06:29:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 182094.1875iter count: 1.0 +[Apr 20, 2026 - 06:29:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 5.1146 iter count: 1.0 +[Apr 20, 2026 - 06:29:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 2442988.5000iter count: 1.0 +[Apr 20, 2026 - 06:29:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6355iter count: 1.0 +[Apr 20, 2026 - 06:29:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.8399 iter count: 1.0 +[Apr 20, 2026 - 06:29:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 15990.0791iter count: 1.0 +[Apr 20, 2026 - 06:29:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 0.8161 iter count: 1.0 +[Apr 20, 2026 - 06:29:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 5.1146 iter count: 1.0 +[Apr 20, 2026 - 06:29:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 2.9285 iter count: 1.0 +[Apr 20, 2026 - 06:29:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 349.6970iter count: 1.0 +[Apr 20, 2026 - 06:29:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.2801 iter count: 1.0 +[Apr 20, 2026 - 06:29:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 5219.8232iter count: 1.0 +[Apr 20, 2026 - 06:29:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[Apr 20, 2026 - 06:29:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 06:30:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121730 43.230595 42.992218 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.742676 69.878906 69.490234 +util 96.750000 99.000000 93.000000 +[Apr 20, 2026 - 06:30:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 261 : avg iteration time 57.19 seconds +[Apr 20, 2026 - 06:30:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 261 : data loading time 0.00 +[Apr 20, 2026 - 06:30:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 261 : avg forward pass time 16.23 +[Apr 20, 2026 - 06:30:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 261 : backward pass time 11.82 +[Apr 20, 2026 - 06:30:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 261 : optimizer step time 1.04 +[Apr 20, 2026 - 06:30:14 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 261-------------------- +[Apr 20, 2026 - 06:30:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0117 iter count: 1.0 +[Apr 20, 2026 - 06:30:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[Apr 20, 2026 - 06:30:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 06:30:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 06:31:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121778 43.230839 42.992199 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.869629 70.054688 69.607422 +util 97.000000 100.000000 92.000000 +[Apr 20, 2026 - 06:31:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 262 : avg iteration time 64.25 seconds +[Apr 20, 2026 - 06:31:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 262 : data loading time 0.00 +[Apr 20, 2026 - 06:31:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 262 : avg forward pass time 19.69 +[Apr 20, 2026 - 06:31:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 262 : backward pass time 11.87 +[Apr 20, 2026 - 06:31:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 262 : optimizer step time 1.05 +[Apr 20, 2026 - 06:31:18 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 262-------------------- +[Apr 20, 2026 - 06:31:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0137 iter count: 1.0 +[Apr 20, 2026 - 06:31:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0137 iter count: 1.0 +[Apr 20, 2026 - 06:31:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 06:31:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 06:32:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121781 43.230839 42.992195 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.825684 69.937500 69.626953 +util 96.500000 100.000000 93.000000 +[Apr 20, 2026 - 06:32:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 263 : avg iteration time 50.31 seconds +[Apr 20, 2026 - 06:32:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 263 : data loading time 0.00 +[Apr 20, 2026 - 06:32:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 263 : avg forward pass time 12.66 +[Apr 20, 2026 - 06:32:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 263 : backward pass time 11.92 +[Apr 20, 2026 - 06:32:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 263 : optimizer step time 1.10 +[Apr 20, 2026 - 06:32:08 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 263-------------------- +[Apr 20, 2026 - 06:32:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0107 iter count: 1.0 +[Apr 20, 2026 - 06:32:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[Apr 20, 2026 - 06:32:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 06:32:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 06:33:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121790 43.230839 42.992218 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.864746 70.054688 69.626953 +util 95.500000 99.000000 92.000000 +[Apr 20, 2026 - 06:33:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 264 : avg iteration time 52.78 seconds +[Apr 20, 2026 - 06:33:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 264 : data loading time 0.00 +[Apr 20, 2026 - 06:33:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 264 : avg forward pass time 13.99 +[Apr 20, 2026 - 06:33:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 264 : backward pass time 11.88 +[Apr 20, 2026 - 06:33:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 264 : optimizer step time 1.05 +[Apr 20, 2026 - 06:33:01 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 264-------------------- +[Apr 20, 2026 - 06:33:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0129 iter count: 1.0 +[Apr 20, 2026 - 06:33:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0129 iter count: 1.0 +[Apr 20, 2026 - 06:33:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 06:33:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 06:36:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121376 43.229237 42.992195 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.507324 135.537109 135.421875 +util 71.250000 78.000000 66.000000 +[Apr 20, 2026 - 06:36:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 265 : avg iteration time 192.91 seconds +[Apr 20, 2026 - 06:36:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 265 : data loading time 0.00 +[Apr 20, 2026 - 06:36:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 265 : avg forward pass time 87.67 +[Apr 20, 2026 - 06:36:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 265 : backward pass time 8.63 +[Apr 20, 2026 - 06:36:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 265 : optimizer step time 0.33 +[Apr 20, 2026 - 06:36:14 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 265-------------------- +[Apr 20, 2026 - 06:36:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 85448.0078 iter count: 1.0 +[Apr 20, 2026 - 06:36:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6159 iter count: 1.0 +[Apr 20, 2026 - 06:36:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 85448.0078iter count: 1.0 +[Apr 20, 2026 - 06:36:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 06:36:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.5237iter count: 1.0 +[Apr 20, 2026 - 06:36:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.7426 iter count: 1.0 +[Apr 20, 2026 - 06:36:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.1214 iter count: 1.0 +[Apr 20, 2026 - 06:36:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 138253.1875iter count: 1.0 +[Apr 20, 2026 - 06:36:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 32.6284iter count: 1.0 +[Apr 20, 2026 - 06:36:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 1533008.2500iter count: 1.0 +[Apr 20, 2026 - 06:36:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6159iter count: 1.0 +[Apr 20, 2026 - 06:36:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 6.0370 iter count: 1.0 +[Apr 20, 2026 - 06:36:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 175255.0312iter count: 1.0 +[Apr 20, 2026 - 06:36:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 2.0090 iter count: 1.0 +[Apr 20, 2026 - 06:36:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 55.5921 iter count: 1.0 +[Apr 20, 2026 - 06:36:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.2978 iter count: 1.0 +[Apr 20, 2026 - 06:36:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 5407.4629iter count: 1.0 +[Apr 20, 2026 - 06:36:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 3.1447 iter count: 1.0 +[Apr 20, 2026 - 06:36:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 538.8403iter count: 1.0 +[Apr 20, 2026 - 06:36:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0134 iter count: 1.0 +[Apr 20, 2026 - 06:36:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 06:37:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121382 43.229259 42.992195 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.879395 69.917969 69.822266 +util 99.000000 100.000000 97.000000 +[Apr 20, 2026 - 06:37:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 266 : avg iteration time 54.28 seconds +[Apr 20, 2026 - 06:37:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 266 : data loading time 0.00 +[Apr 20, 2026 - 06:37:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 266 : avg forward pass time 14.80 +[Apr 20, 2026 - 06:37:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 266 : backward pass time 11.77 +[Apr 20, 2026 - 06:37:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 266 : optimizer step time 0.99 +[Apr 20, 2026 - 06:37:08 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 266-------------------- +[Apr 20, 2026 - 06:37:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0123 iter count: 1.0 +[Apr 20, 2026 - 06:37:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0123 iter count: 1.0 +[Apr 20, 2026 - 06:37:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 06:37:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 06:38:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121826 43.231007 42.992203 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.840332 69.917969 69.646484 +util 98.000000 100.000000 94.000000 +[Apr 20, 2026 - 06:38:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 267 : avg iteration time 55.71 seconds +[Apr 20, 2026 - 06:38:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 267 : data loading time 0.00 +[Apr 20, 2026 - 06:38:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 267 : avg forward pass time 15.38 +[Apr 20, 2026 - 06:38:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 267 : backward pass time 11.92 +[Apr 20, 2026 - 06:38:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 267 : optimizer step time 1.16 +[Apr 20, 2026 - 06:38:04 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 267-------------------- +[Apr 20, 2026 - 06:38:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0121 iter count: 1.0 +[Apr 20, 2026 - 06:38:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[Apr 20, 2026 - 06:38:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 06:38:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 06:38:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121821 43.230995 42.992195 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.835449 69.917969 69.666016 +util 98.750000 100.000000 96.000000 +[Apr 20, 2026 - 06:38:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 268 : avg iteration time 53.28 seconds +[Apr 20, 2026 - 06:38:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 268 : data loading time 0.00 +[Apr 20, 2026 - 06:38:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 268 : avg forward pass time 14.11 +[Apr 20, 2026 - 06:38:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 268 : backward pass time 11.92 +[Apr 20, 2026 - 06:38:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 268 : optimizer step time 1.22 +[Apr 20, 2026 - 06:38:57 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 268-------------------- +[Apr 20, 2026 - 06:38:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0099 iter count: 1.0 +[Apr 20, 2026 - 06:38:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[Apr 20, 2026 - 06:38:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 06:38:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 06:39:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121818 43.230995 42.992195 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.845215 69.996094 69.646484 +util 98.000000 100.000000 94.000000 +[Apr 20, 2026 - 06:39:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 269 : avg iteration time 58.38 seconds +[Apr 20, 2026 - 06:39:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 269 : data loading time 0.00 +[Apr 20, 2026 - 06:39:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 269 : avg forward pass time 16.80 +[Apr 20, 2026 - 06:39:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 269 : backward pass time 11.84 +[Apr 20, 2026 - 06:39:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 269 : optimizer step time 1.10 +[Apr 20, 2026 - 06:39:56 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 269-------------------- +[Apr 20, 2026 - 06:39:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0129 iter count: 1.0 +[Apr 20, 2026 - 06:39:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0129 iter count: 1.0 +[Apr 20, 2026 - 06:39:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 06:39:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 06:42:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121484 43.229633 42.992195 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.493652 135.517578 135.421875 +util 70.250000 77.000000 64.000000 +[Apr 20, 2026 - 06:42:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 270 : avg iteration time 173.58 seconds +[Apr 20, 2026 - 06:42:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 270 : data loading time 0.00 +[Apr 20, 2026 - 06:42:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 270 : avg forward pass time 77.98 +[Apr 20, 2026 - 06:42:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 270 : backward pass time 8.68 +[Apr 20, 2026 - 06:42:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 270 : optimizer step time 0.38 +[Apr 20, 2026 - 06:42:49 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 270-------------------- +[Apr 20, 2026 - 06:42:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 53224.6527 iter count: 1.0 +[Apr 20, 2026 - 06:42:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5319 iter count: 1.0 +[Apr 20, 2026 - 06:42:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 53224.6527iter count: 1.0 +[Apr 20, 2026 - 06:42:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 06:42:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.1404iter count: 1.0 +[Apr 20, 2026 - 06:42:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.7832 iter count: 1.0 +[Apr 20, 2026 - 06:42:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.2035 iter count: 1.0 +[Apr 20, 2026 - 06:42:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 139199.7812iter count: 1.0 +[Apr 20, 2026 - 06:42:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 4.7895 iter count: 1.0 +[Apr 20, 2026 - 06:42:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 1806590.5000iter count: 1.0 +[Apr 20, 2026 - 06:42:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5319iter count: 1.0 +[Apr 20, 2026 - 06:42:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.2469 iter count: 1.0 +[Apr 20, 2026 - 06:42:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 661.0691iter count: 1.0 +[Apr 20, 2026 - 06:42:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 6.2876 iter count: 1.0 +[Apr 20, 2026 - 06:42:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 289293.8438iter count: 1.0 +[Apr 20, 2026 - 06:42:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.6249 iter count: 1.0 +[Apr 20, 2026 - 06:42:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 10403.2354iter count: 1.0 +[Apr 20, 2026 - 06:42:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.5779 iter count: 1.0 +[Apr 20, 2026 - 06:42:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 9469.1836iter count: 1.0 +[Apr 20, 2026 - 06:42:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[Apr 20, 2026 - 06:42:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 06:43:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121671 43.230381 42.992195 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.728027 69.839844 69.470703 +util 96.500000 100.000000 91.000000 +[Apr 20, 2026 - 06:43:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 271 : avg iteration time 54.39 seconds +[Apr 20, 2026 - 06:43:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 271 : data loading time 0.00 +[Apr 20, 2026 - 06:43:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 271 : avg forward pass time 14.76 +[Apr 20, 2026 - 06:43:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 271 : backward pass time 11.93 +[Apr 20, 2026 - 06:43:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 271 : optimizer step time 1.11 +[Apr 20, 2026 - 06:43:44 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 271-------------------- +[Apr 20, 2026 - 06:43:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0093 iter count: 1.0 +[Apr 20, 2026 - 06:43:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[Apr 20, 2026 - 06:43:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 06:43:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 06:44:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121800 43.230881 42.992195 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.791504 69.878906 69.646484 +util 97.250000 100.000000 92.000000 +[Apr 20, 2026 - 06:44:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 272 : avg iteration time 50.08 seconds +[Apr 20, 2026 - 06:44:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 272 : data loading time 0.00 +[Apr 20, 2026 - 06:44:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 272 : avg forward pass time 12.59 +[Apr 20, 2026 - 06:44:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 272 : backward pass time 11.86 +[Apr 20, 2026 - 06:44:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 272 : optimizer step time 1.21 +[Apr 20, 2026 - 06:44:34 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 272-------------------- +[Apr 20, 2026 - 06:44:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0100 iter count: 1.0 +[Apr 20, 2026 - 06:44:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[Apr 20, 2026 - 06:44:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 06:44:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 06:45:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121791 43.230885 42.992195 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.771973 69.839844 69.626953 +util 97.500000 100.000000 93.000000 +[Apr 20, 2026 - 06:45:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 273 : avg iteration time 47.10 seconds +[Apr 20, 2026 - 06:45:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 273 : data loading time 0.00 +[Apr 20, 2026 - 06:45:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 273 : avg forward pass time 11.22 +[Apr 20, 2026 - 06:45:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 273 : backward pass time 11.81 +[Apr 20, 2026 - 06:45:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 273 : optimizer step time 1.02 +[Apr 20, 2026 - 06:45:21 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 273-------------------- +[Apr 20, 2026 - 06:45:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0123 iter count: 1.0 +[Apr 20, 2026 - 06:45:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0123 iter count: 1.0 +[Apr 20, 2026 - 06:45:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 06:45:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 06:46:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121791 43.230885 42.992195 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.791504 69.937500 69.509766 +util 97.500000 99.000000 94.000000 +[Apr 20, 2026 - 06:46:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 274 : avg iteration time 64.41 seconds +[Apr 20, 2026 - 06:46:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 274 : data loading time 0.00 +[Apr 20, 2026 - 06:46:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 274 : avg forward pass time 19.80 +[Apr 20, 2026 - 06:46:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 274 : backward pass time 11.85 +[Apr 20, 2026 - 06:46:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 274 : optimizer step time 1.04 +[Apr 20, 2026 - 06:46:25 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 274-------------------- +[Apr 20, 2026 - 06:46:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0128 iter count: 1.0 +[Apr 20, 2026 - 06:46:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0128 iter count: 1.0 +[Apr 20, 2026 - 06:46:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 06:46:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 06:49:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121640 43.230122 42.992207 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.457520 135.476562 135.400391 +util 71.000000 79.000000 65.000000 +[Apr 20, 2026 - 06:49:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 275 : avg iteration time 185.36 seconds +[Apr 20, 2026 - 06:49:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 275 : data loading time 0.00 +[Apr 20, 2026 - 06:49:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 275 : avg forward pass time 83.83 +[Apr 20, 2026 - 06:49:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 275 : backward pass time 8.70 +[Apr 20, 2026 - 06:49:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 275 : optimizer step time 0.33 +[Apr 20, 2026 - 06:49:31 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 275-------------------- +[Apr 20, 2026 - 06:49:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 534581.7754 iter count: 1.0 +[Apr 20, 2026 - 06:49:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.4805 iter count: 1.0 +[Apr 20, 2026 - 06:49:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 534581.7754iter count: 1.0 +[Apr 20, 2026 - 06:49:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 06:49:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.4550iter count: 1.0 +[Apr 20, 2026 - 06:49:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.1866 iter count: 1.0 +[Apr 20, 2026 - 06:49:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 8.5499 iter count: 1.0 +[Apr 20, 2026 - 06:49:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 1068528.2500iter count: 1.0 +[Apr 20, 2026 - 06:49:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 10.7323iter count: 1.0 +[Apr 20, 2026 - 06:49:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 26687930.0000iter count: 1.0 +[Apr 20, 2026 - 06:49:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.4805iter count: 1.0 +[Apr 20, 2026 - 06:49:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.8159 iter count: 1.0 +[Apr 20, 2026 - 06:49:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 2062.8574iter count: 1.0 +[Apr 20, 2026 - 06:49:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.3684 iter count: 1.0 +[Apr 20, 2026 - 06:49:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 842.8136iter count: 1.0 +[Apr 20, 2026 - 06:49:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.7063 iter count: 1.0 +[Apr 20, 2026 - 06:49:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 12242.1670iter count: 1.0 +[Apr 20, 2026 - 06:49:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 5.6495 iter count: 1.0 +[Apr 20, 2026 - 06:49:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 80734.8281iter count: 1.0 +[Apr 20, 2026 - 06:49:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[Apr 20, 2026 - 06:49:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 06:50:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121694 43.230370 42.992207 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.767090 69.839844 69.626953 +util 98.500000 100.000000 96.000000 +[Apr 20, 2026 - 06:50:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 276 : avg iteration time 51.81 seconds +[Apr 20, 2026 - 06:50:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 276 : data loading time 0.00 +[Apr 20, 2026 - 06:50:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 276 : avg forward pass time 13.37 +[Apr 20, 2026 - 06:50:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 276 : backward pass time 11.93 +[Apr 20, 2026 - 06:50:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 276 : optimizer step time 1.23 +[Apr 20, 2026 - 06:50:23 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 276-------------------- +[Apr 20, 2026 - 06:50:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0095 iter count: 1.0 +[Apr 20, 2026 - 06:50:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[Apr 20, 2026 - 06:50:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 06:50:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 06:51:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121691 43.230370 42.992207 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.752441 69.820312 69.626953 +util 97.750000 100.000000 94.000000 +[Apr 20, 2026 - 06:51:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 277 : avg iteration time 58.64 seconds +[Apr 20, 2026 - 06:51:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 277 : data loading time 0.00 +[Apr 20, 2026 - 06:51:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 277 : avg forward pass time 16.89 +[Apr 20, 2026 - 06:51:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 277 : backward pass time 11.91 +[Apr 20, 2026 - 06:51:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 277 : optimizer step time 1.09 +[Apr 20, 2026 - 06:51:21 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 277-------------------- +[Apr 20, 2026 - 06:51:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0099 iter count: 1.0 +[Apr 20, 2026 - 06:51:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[Apr 20, 2026 - 06:51:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 06:51:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 06:52:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121756 43.230621 42.992207 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.806152 69.878906 69.724609 +util 98.750000 100.000000 95.000000 +[Apr 20, 2026 - 06:52:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 278 : avg iteration time 53.26 seconds +[Apr 20, 2026 - 06:52:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 278 : data loading time 0.00 +[Apr 20, 2026 - 06:52:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 278 : avg forward pass time 14.09 +[Apr 20, 2026 - 06:52:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 278 : backward pass time 12.02 +[Apr 20, 2026 - 06:52:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 278 : optimizer step time 1.19 +[Apr 20, 2026 - 06:52:14 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 278-------------------- +[Apr 20, 2026 - 06:52:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0105 iter count: 1.0 +[Apr 20, 2026 - 06:52:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[Apr 20, 2026 - 06:52:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 06:52:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 06:53:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121822 43.230873 42.992207 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.801270 69.878906 69.705078 +util 98.250000 100.000000 95.000000 +[Apr 20, 2026 - 06:53:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 279 : avg iteration time 58.62 seconds +[Apr 20, 2026 - 06:53:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 279 : data loading time 0.00 +[Apr 20, 2026 - 06:53:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 279 : avg forward pass time 16.88 +[Apr 20, 2026 - 06:53:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 279 : backward pass time 11.91 +[Apr 20, 2026 - 06:53:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 279 : optimizer step time 1.08 +[Apr 20, 2026 - 06:53:13 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 279-------------------- +[Apr 20, 2026 - 06:53:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0114 iter count: 1.0 +[Apr 20, 2026 - 06:53:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[Apr 20, 2026 - 06:53:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 06:53:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 06:56:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121675 43.229733 42.992439 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.497559 135.515625 135.478516 +util 70.250000 78.000000 65.000000 +[Apr 20, 2026 - 06:56:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 280 : avg iteration time 181.47 seconds +[Apr 20, 2026 - 06:56:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 280 : data loading time 0.00 +[Apr 20, 2026 - 06:56:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 280 : avg forward pass time 81.97 +[Apr 20, 2026 - 06:56:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 280 : backward pass time 8.64 +[Apr 20, 2026 - 06:56:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 280 : optimizer step time 0.33 +[Apr 20, 2026 - 06:56:14 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 280-------------------- +[Apr 20, 2026 - 06:56:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 62591.0073 iter count: 1.0 +[Apr 20, 2026 - 06:56:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6535 iter count: 1.0 +[Apr 20, 2026 - 06:56:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 62591.0073iter count: 1.0 +[Apr 20, 2026 - 06:56:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 06:56:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.7667iter count: 1.0 +[Apr 20, 2026 - 06:56:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.5549 iter count: 1.0 +[Apr 20, 2026 - 06:56:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.9246 iter count: 1.0 +[Apr 20, 2026 - 06:56:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 122418.4375iter count: 1.0 +[Apr 20, 2026 - 06:56:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 22.4152iter count: 1.0 +[Apr 20, 2026 - 06:56:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 1034167.9375iter count: 1.0 +[Apr 20, 2026 - 06:56:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6535iter count: 1.0 +[Apr 20, 2026 - 06:56:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.1519 iter count: 1.0 +[Apr 20, 2026 - 06:56:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 4039.0330iter count: 1.0 +[Apr 20, 2026 - 06:56:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.9981 iter count: 1.0 +[Apr 20, 2026 - 06:56:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 21943.1875iter count: 1.0 +[Apr 20, 2026 - 06:56:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 6.8000 iter count: 1.0 +[Apr 20, 2026 - 06:56:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 806170.0000iter count: 1.0 +[Apr 20, 2026 - 06:56:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 3.8593 iter count: 1.0 +[Apr 20, 2026 - 06:56:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 2249.6001iter count: 1.0 +[Apr 20, 2026 - 06:56:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[Apr 20, 2026 - 06:56:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 06:57:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121676 43.229733 42.992420 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.645020 69.800781 69.468750 +util 98.500000 100.000000 96.000000 +[Apr 20, 2026 - 06:57:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 281 : avg iteration time 51.73 seconds +[Apr 20, 2026 - 06:57:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 281 : data loading time 0.00 +[Apr 20, 2026 - 06:57:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 281 : avg forward pass time 13.38 +[Apr 20, 2026 - 06:57:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 281 : backward pass time 11.94 +[Apr 20, 2026 - 06:57:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 281 : optimizer step time 1.12 +[Apr 20, 2026 - 06:57:06 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 281-------------------- +[Apr 20, 2026 - 06:57:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0090 iter count: 1.0 +[Apr 20, 2026 - 06:57:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[Apr 20, 2026 - 06:57:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 06:57:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 06:58:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122046 43.231232 42.992420 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.737793 69.937500 69.468750 +util 98.250000 100.000000 95.000000 +[Apr 20, 2026 - 06:58:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 282 : avg iteration time 55.67 seconds +[Apr 20, 2026 - 06:58:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 282 : data loading time 0.00 +[Apr 20, 2026 - 06:58:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 282 : avg forward pass time 15.44 +[Apr 20, 2026 - 06:58:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 282 : backward pass time 11.84 +[Apr 20, 2026 - 06:58:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 282 : optimizer step time 1.11 +[Apr 20, 2026 - 06:58:02 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 282-------------------- +[Apr 20, 2026 - 06:58:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0086 iter count: 1.0 +[Apr 20, 2026 - 06:58:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[Apr 20, 2026 - 06:58:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 06:58:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 06:58:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122050 43.231228 42.992420 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.684082 69.898438 69.449219 +util 96.750000 99.000000 94.000000 +[Apr 20, 2026 - 06:58:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 283 : avg iteration time 50.00 seconds +[Apr 20, 2026 - 06:58:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 283 : data loading time 0.00 +[Apr 20, 2026 - 06:58:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 283 : avg forward pass time 12.65 +[Apr 20, 2026 - 06:58:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 283 : backward pass time 11.79 +[Apr 20, 2026 - 06:58:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 283 : optimizer step time 1.03 +[Apr 20, 2026 - 06:58:52 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 283-------------------- +[Apr 20, 2026 - 06:58:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0086 iter count: 1.0 +[Apr 20, 2026 - 06:58:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[Apr 20, 2026 - 06:58:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 06:58:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 06:59:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122050 43.231251 42.992420 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.708496 69.957031 69.429688 +util 97.750000 100.000000 94.000000 +[Apr 20, 2026 - 06:59:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 284 : avg iteration time 50.04 seconds +[Apr 20, 2026 - 06:59:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 284 : data loading time 0.00 +[Apr 20, 2026 - 06:59:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 284 : avg forward pass time 12.69 +[Apr 20, 2026 - 06:59:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 284 : backward pass time 11.82 +[Apr 20, 2026 - 06:59:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 284 : optimizer step time 1.02 +[Apr 20, 2026 - 06:59:42 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 284-------------------- +[Apr 20, 2026 - 06:59:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0090 iter count: 1.0 +[Apr 20, 2026 - 06:59:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[Apr 20, 2026 - 06:59:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 06:59:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 07:02:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121638 43.229603 42.992420 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.518066 135.537109 135.498047 +util 67.500000 72.000000 61.000000 +[Apr 20, 2026 - 07:02:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 285 : avg iteration time 189.44 seconds +[Apr 20, 2026 - 07:02:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 285 : data loading time 0.00 +[Apr 20, 2026 - 07:02:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 285 : avg forward pass time 85.92 +[Apr 20, 2026 - 07:02:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 285 : backward pass time 8.63 +[Apr 20, 2026 - 07:02:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 285 : optimizer step time 0.34 +[Apr 20, 2026 - 07:02:51 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 285-------------------- +[Apr 20, 2026 - 07:02:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 212394.6602 iter count: 1.0 +[Apr 20, 2026 - 07:02:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6754 iter count: 1.0 +[Apr 20, 2026 - 07:02:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 212394.6602iter count: 1.0 +[Apr 20, 2026 - 07:02:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 07:02:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.6833iter count: 1.0 +[Apr 20, 2026 - 07:02:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 2.4648 iter count: 1.0 +[Apr 20, 2026 - 07:02:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.6523 iter count: 1.0 +[Apr 20, 2026 - 07:02:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 311980.7500iter count: 1.0 +[Apr 20, 2026 - 07:02:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 138.3276iter count: 1.0 +[Apr 20, 2026 - 07:02:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 4432730.5000iter count: 1.0 +[Apr 20, 2026 - 07:02:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6754iter count: 1.0 +[Apr 20, 2026 - 07:02:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 7.2563 iter count: 1.0 +[Apr 20, 2026 - 07:02:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 2007967.1250iter count: 1.0 +[Apr 20, 2026 - 07:02:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 5.1435 iter count: 1.0 +[Apr 20, 2026 - 07:02:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 29348.4473iter count: 1.0 +[Apr 20, 2026 - 07:02:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 3.9005 iter count: 1.0 +[Apr 20, 2026 - 07:02:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 2442.9707iter count: 1.0 +[Apr 20, 2026 - 07:02:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.4366 iter count: 1.0 +[Apr 20, 2026 - 07:02:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 7138.1509iter count: 1.0 +[Apr 20, 2026 - 07:02:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[Apr 20, 2026 - 07:02:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 07:03:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121701 43.229855 42.992420 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.664551 69.761719 69.412109 +util 96.500000 100.000000 91.000000 +[Apr 20, 2026 - 07:03:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 286 : avg iteration time 59.79 seconds +[Apr 20, 2026 - 07:03:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 286 : data loading time 0.00 +[Apr 20, 2026 - 07:03:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 286 : avg forward pass time 17.51 +[Apr 20, 2026 - 07:03:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 286 : backward pass time 11.82 +[Apr 20, 2026 - 07:03:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 286 : optimizer step time 1.04 +[Apr 20, 2026 - 07:03:51 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 286-------------------- +[Apr 20, 2026 - 07:03:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0102 iter count: 1.0 +[Apr 20, 2026 - 07:03:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[Apr 20, 2026 - 07:03:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 07:03:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 07:04:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121950 43.230846 42.992424 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.698730 69.839844 69.470703 +util 96.250000 99.000000 93.000000 +[Apr 20, 2026 - 07:04:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 287 : avg iteration time 58.33 seconds +[Apr 20, 2026 - 07:04:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 287 : data loading time 0.00 +[Apr 20, 2026 - 07:04:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 287 : avg forward pass time 16.84 +[Apr 20, 2026 - 07:04:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 287 : backward pass time 11.79 +[Apr 20, 2026 - 07:04:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 287 : optimizer step time 1.06 +[Apr 20, 2026 - 07:04:49 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 287-------------------- +[Apr 20, 2026 - 07:04:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0087 iter count: 1.0 +[Apr 20, 2026 - 07:04:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[Apr 20, 2026 - 07:04:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 07:04:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 07:05:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121951 43.230846 42.992420 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.649902 69.742188 69.412109 +util 98.250000 100.000000 94.000000 +[Apr 20, 2026 - 07:05:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 288 : avg iteration time 47.30 seconds +[Apr 20, 2026 - 07:05:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 288 : data loading time 0.00 +[Apr 20, 2026 - 07:05:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 288 : avg forward pass time 11.26 +[Apr 20, 2026 - 07:05:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 288 : backward pass time 11.89 +[Apr 20, 2026 - 07:05:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 288 : optimizer step time 1.08 +[Apr 20, 2026 - 07:05:37 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 288-------------------- +[Apr 20, 2026 - 07:05:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0090 iter count: 1.0 +[Apr 20, 2026 - 07:05:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[Apr 20, 2026 - 07:05:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 07:05:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 07:06:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122013 43.231098 42.992420 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.713379 69.742188 69.626953 +util 97.250000 100.000000 91.000000 +[Apr 20, 2026 - 07:06:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 289 : avg iteration time 61.07 seconds +[Apr 20, 2026 - 07:06:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 289 : data loading time 0.00 +[Apr 20, 2026 - 07:06:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 289 : avg forward pass time 18.20 +[Apr 20, 2026 - 07:06:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 289 : backward pass time 11.78 +[Apr 20, 2026 - 07:06:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 289 : optimizer step time 1.06 +[Apr 20, 2026 - 07:06:38 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 289-------------------- +[Apr 20, 2026 - 07:06:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0098 iter count: 1.0 +[Apr 20, 2026 - 07:06:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[Apr 20, 2026 - 07:06:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 07:06:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 07:09:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121757 43.230072 42.992420 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.467773 135.496094 135.421875 +util 70.750000 74.000000 65.000000 +[Apr 20, 2026 - 07:09:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 290 : avg iteration time 186.93 seconds +[Apr 20, 2026 - 07:09:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 290 : data loading time 0.00 +[Apr 20, 2026 - 07:09:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 290 : avg forward pass time 84.70 +[Apr 20, 2026 - 07:09:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 290 : backward pass time 8.64 +[Apr 20, 2026 - 07:09:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 290 : optimizer step time 0.33 +[Apr 20, 2026 - 07:09:45 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 290-------------------- +[Apr 20, 2026 - 07:09:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 100356.5879 iter count: 1.0 +[Apr 20, 2026 - 07:09:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6392 iter count: 1.0 +[Apr 20, 2026 - 07:09:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 100356.5879iter count: 1.0 +[Apr 20, 2026 - 07:09:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 07:09:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.7123iter count: 1.0 +[Apr 20, 2026 - 07:09:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 2.0151 iter count: 1.0 +[Apr 20, 2026 - 07:09:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.2595 iter count: 1.0 +[Apr 20, 2026 - 07:09:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 193348.1250iter count: 1.0 +[Apr 20, 2026 - 07:09:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 56.2731iter count: 1.0 +[Apr 20, 2026 - 07:09:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 2020774.6250iter count: 1.0 +[Apr 20, 2026 - 07:09:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6392iter count: 1.0 +[Apr 20, 2026 - 07:09:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.3634 iter count: 1.0 +[Apr 20, 2026 - 07:09:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 6166.0186iter count: 1.0 +[Apr 20, 2026 - 07:09:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 2.1782 iter count: 1.0 +[Apr 20, 2026 - 07:09:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 77.9732 iter count: 1.0 +[Apr 20, 2026 - 07:09:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 3.1750 iter count: 1.0 +[Apr 20, 2026 - 07:09:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 572.5322iter count: 1.0 +[Apr 20, 2026 - 07:09:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.2062 iter count: 1.0 +[Apr 20, 2026 - 07:09:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 4502.2158iter count: 1.0 +[Apr 20, 2026 - 07:09:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[Apr 20, 2026 - 07:09:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 07:10:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121886 43.230568 42.992420 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.669434 69.820312 69.451172 +util 96.500000 100.000000 92.000000 +[Apr 20, 2026 - 07:10:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 291 : avg iteration time 57.29 seconds +[Apr 20, 2026 - 07:10:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 291 : data loading time 0.00 +[Apr 20, 2026 - 07:10:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 291 : avg forward pass time 16.11 +[Apr 20, 2026 - 07:10:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 291 : backward pass time 11.86 +[Apr 20, 2026 - 07:10:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 291 : optimizer step time 1.24 +[Apr 20, 2026 - 07:10:42 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 291-------------------- +[Apr 20, 2026 - 07:10:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0099 iter count: 1.0 +[Apr 20, 2026 - 07:10:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[Apr 20, 2026 - 07:10:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 07:10:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 07:11:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121886 43.230568 42.992420 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.640137 69.742188 69.470703 +util 96.500000 99.000000 93.000000 +[Apr 20, 2026 - 07:11:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 292 : avg iteration time 55.65 seconds +[Apr 20, 2026 - 07:11:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 292 : data loading time 0.00 +[Apr 20, 2026 - 07:11:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 292 : avg forward pass time 15.47 +[Apr 20, 2026 - 07:11:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 292 : backward pass time 11.79 +[Apr 20, 2026 - 07:11:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 292 : optimizer step time 1.04 +[Apr 20, 2026 - 07:11:38 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 292-------------------- +[Apr 20, 2026 - 07:11:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0098 iter count: 1.0 +[Apr 20, 2026 - 07:11:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[Apr 20, 2026 - 07:11:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 07:11:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 07:12:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121884 43.230568 42.992435 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.732910 69.898438 69.451172 +util 96.250000 99.000000 93.000000 +[Apr 20, 2026 - 07:12:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 293 : avg iteration time 52.76 seconds +[Apr 20, 2026 - 07:12:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 293 : data loading time 0.00 +[Apr 20, 2026 - 07:12:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 293 : avg forward pass time 14.07 +[Apr 20, 2026 - 07:12:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 293 : backward pass time 11.81 +[Apr 20, 2026 - 07:12:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 293 : optimizer step time 1.04 +[Apr 20, 2026 - 07:12:30 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 293-------------------- +[Apr 20, 2026 - 07:12:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0083 iter count: 1.0 +[Apr 20, 2026 - 07:12:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[Apr 20, 2026 - 07:12:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 07:12:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 07:13:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121886 43.230568 42.992420 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.723145 69.917969 69.451172 +util 97.000000 100.000000 92.000000 +[Apr 20, 2026 - 07:13:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 294 : avg iteration time 64.09 seconds +[Apr 20, 2026 - 07:13:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 294 : data loading time 0.00 +[Apr 20, 2026 - 07:13:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 294 : avg forward pass time 19.66 +[Apr 20, 2026 - 07:13:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 294 : backward pass time 11.87 +[Apr 20, 2026 - 07:13:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 294 : optimizer step time 1.08 +[Apr 20, 2026 - 07:13:35 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 294-------------------- +[Apr 20, 2026 - 07:13:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0109 iter count: 1.0 +[Apr 20, 2026 - 07:13:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[Apr 20, 2026 - 07:13:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 07:13:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 07:16:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121678 43.229759 42.992420 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.507324 135.554688 135.439453 +util 71.750000 75.000000 66.000000 +[Apr 20, 2026 - 07:16:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 295 : avg iteration time 184.24 seconds +[Apr 20, 2026 - 07:16:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 295 : data loading time 0.00 +[Apr 20, 2026 - 07:16:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 295 : avg forward pass time 83.34 +[Apr 20, 2026 - 07:16:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 295 : backward pass time 8.36 +[Apr 20, 2026 - 07:16:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 295 : optimizer step time 0.33 +[Apr 20, 2026 - 07:16:39 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 295-------------------- +[Apr 20, 2026 - 07:16:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 434370.6411 iter count: 1.0 +[Apr 20, 2026 - 07:16:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6533 iter count: 1.0 +[Apr 20, 2026 - 07:16:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 434370.6411iter count: 1.0 +[Apr 20, 2026 - 07:16:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 07:16:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.1665iter count: 1.0 +[Apr 20, 2026 - 07:16:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.2739 iter count: 1.0 +[Apr 20, 2026 - 07:16:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 8.3900 iter count: 1.0 +[Apr 20, 2026 - 07:16:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 706151.5000iter count: 1.0 +[Apr 20, 2026 - 07:16:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 12.7795iter count: 1.0 +[Apr 20, 2026 - 07:16:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 19383302.0000iter count: 1.0 +[Apr 20, 2026 - 07:16:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6533iter count: 1.0 +[Apr 20, 2026 - 07:16:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.5877 iter count: 1.0 +[Apr 20, 2026 - 07:16:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 9656.7568iter count: 1.0 +[Apr 20, 2026 - 07:16:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 2.5738 iter count: 1.0 +[Apr 20, 2026 - 07:16:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 172.0323iter count: 1.0 +[Apr 20, 2026 - 07:16:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 2.9227 iter count: 1.0 +[Apr 20, 2026 - 07:16:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 345.6378iter count: 1.0 +[Apr 20, 2026 - 07:16:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 3.4663 iter count: 1.0 +[Apr 20, 2026 - 07:16:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 1025.0934iter count: 1.0 +[Apr 20, 2026 - 07:16:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[Apr 20, 2026 - 07:16:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 07:17:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121808 43.230259 42.992420 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.928223 70.035156 69.878906 +util 97.250000 100.000000 92.000000 +[Apr 20, 2026 - 07:17:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 296 : avg iteration time 65.96 seconds +[Apr 20, 2026 - 07:17:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 296 : data loading time 0.00 +[Apr 20, 2026 - 07:17:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 296 : avg forward pass time 19.91 +[Apr 20, 2026 - 07:17:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 296 : backward pass time 11.82 +[Apr 20, 2026 - 07:17:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 296 : optimizer step time 1.49 +[Apr 20, 2026 - 07:17:45 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 296-------------------- +[Apr 20, 2026 - 07:17:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0108 iter count: 1.0 +[Apr 20, 2026 - 07:17:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[Apr 20, 2026 - 07:17:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 07:17:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 07:18:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122055 43.231266 42.992420 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.752441 69.898438 69.587891 +util 97.750000 99.000000 94.000000 +[Apr 20, 2026 - 07:18:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 297 : avg iteration time 58.02 seconds +[Apr 20, 2026 - 07:18:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 297 : data loading time 0.00 +[Apr 20, 2026 - 07:18:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 297 : avg forward pass time 16.93 +[Apr 20, 2026 - 07:18:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 297 : backward pass time 11.56 +[Apr 20, 2026 - 07:18:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 297 : optimizer step time 0.99 +[Apr 20, 2026 - 07:18:43 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 297-------------------- +[Apr 20, 2026 - 07:18:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0085 iter count: 1.0 +[Apr 20, 2026 - 07:18:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[Apr 20, 2026 - 07:18:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 07:18:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 07:19:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122058 43.231266 42.992432 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.742676 69.898438 69.587891 +util 97.500000 100.000000 92.000000 +[Apr 20, 2026 - 07:19:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 298 : avg iteration time 50.26 seconds +[Apr 20, 2026 - 07:19:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 298 : data loading time 0.00 +[Apr 20, 2026 - 07:19:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 298 : avg forward pass time 12.43 +[Apr 20, 2026 - 07:19:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 298 : backward pass time 11.57 +[Apr 20, 2026 - 07:19:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 298 : optimizer step time 0.97 +[Apr 20, 2026 - 07:19:33 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 298-------------------- +[Apr 20, 2026 - 07:19:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0090 iter count: 1.0 +[Apr 20, 2026 - 07:19:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[Apr 20, 2026 - 07:19:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 07:19:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 07:20:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122044 43.231201 42.992420 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.372559 71.873047 69.820312 +util 97.500000 100.000000 93.000000 +[Apr 20, 2026 - 07:20:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 299 : avg iteration time 53.95 seconds +[Apr 20, 2026 - 07:20:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 299 : data loading time 0.00 +[Apr 20, 2026 - 07:20:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 299 : avg forward pass time 14.19 +[Apr 20, 2026 - 07:20:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 299 : backward pass time 12.77 +[Apr 20, 2026 - 07:20:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 299 : optimizer step time 1.26 +[Apr 20, 2026 - 07:20:27 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 299-------------------- +[Apr 20, 2026 - 07:20:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0089 iter count: 1.0 +[Apr 20, 2026 - 07:20:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[Apr 20, 2026 - 07:20:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 07:20:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 07:23:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121903 43.230659 42.992420 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.488281 135.517578 135.458984 +util 71.000000 78.000000 61.000000 +[Apr 20, 2026 - 07:23:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 300 : avg iteration time 187.66 seconds +[Apr 20, 2026 - 07:23:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 300 : data loading time 0.00 +[Apr 20, 2026 - 07:23:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 300 : avg forward pass time 85.35 +[Apr 20, 2026 - 07:23:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 300 : backward pass time 8.34 +[Apr 20, 2026 - 07:23:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 300 : optimizer step time 0.34 +[Apr 20, 2026 - 07:23:35 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 300-------------------- +[Apr 20, 2026 - 07:23:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 495395.8936 iter count: 1.0 +[Apr 20, 2026 - 07:23:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6427 iter count: 1.0 +[Apr 20, 2026 - 07:23:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 495395.8936iter count: 1.0 +[Apr 20, 2026 - 07:23:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 07:23:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.5237iter count: 1.0 +[Apr 20, 2026 - 07:23:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.5104 iter count: 1.0 +[Apr 20, 2026 - 07:23:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 8.2095 iter count: 1.0 +[Apr 20, 2026 - 07:23:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 602209.1250iter count: 1.0 +[Apr 20, 2026 - 07:23:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 2.7754 iter count: 1.0 +[Apr 20, 2026 - 07:23:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 13510217.0000iter count: 1.0 +[Apr 20, 2026 - 07:23:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6427iter count: 1.0 +[Apr 20, 2026 - 07:23:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 7.0714 iter count: 1.0 +[Apr 20, 2026 - 07:23:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 1387273.5000iter count: 1.0 +[Apr 20, 2026 - 07:23:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 7.2105 iter count: 1.0 +[Apr 20, 2026 - 07:23:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 1832226.5000iter count: 1.0 +[Apr 20, 2026 - 07:23:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 6.2058 iter count: 1.0 +[Apr 20, 2026 - 07:23:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 245628.9219iter count: 1.0 +[Apr 20, 2026 - 07:23:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 6.7938 iter count: 1.0 +[Apr 20, 2026 - 07:23:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 796132.1875iter count: 1.0 +[Apr 20, 2026 - 07:23:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[Apr 20, 2026 - 07:23:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 21.49 seconds +[Apr 20, 2026 - 07:23:56 | INFO | fastgen.trainer:save_checkpoint:266 ] Saving checkpoint iteration 300 +[Apr 20, 2026 - 07:23:56 | INFO | fastgen.utils.checkpointer:save:301 ] Saving FSDP model to prefix /tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000300 +[Apr 20, 2026 - 07:24:13 | SUCCESS | fastgen.utils.checkpointer:save:329 ] Model saved at iteration 300 +[Apr 20, 2026 - 07:24:13 | INFO | fastgen.trainer:validate:358 ] Validation iteration 300 +[Apr 20, 2026 - 07:24:13 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 07:24:17 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 3.3s +[Apr 20, 2026 - 07:24:20 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 07:24:23 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 2.9s +[Apr 20, 2026 - 07:24:26 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 07:24:29 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 2.9s +[Apr 20, 2026 - 07:24:33 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 07:24:35 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 2.9s +[Apr 20, 2026 - 07:24:39 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 07:24:42 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 2.9s +[Apr 20, 2026 - 07:24:45 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 07:24:48 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 2.9s +[Apr 20, 2026 - 07:24:51 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 07:24:54 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 2.9s +[Apr 20, 2026 - 07:24:57 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 07:25:00 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 2.9s +[Apr 20, 2026 - 07:25:04 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 07:25:07 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 2.9s +[Apr 20, 2026 - 07:25:10 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 07:25:13 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 2.9s +[Apr 20, 2026 - 07:25:16 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging val0 stats at iteration 300-------------------- +[Apr 20, 2026 - 07:25:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0000 iter count: 10.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[Apr 20, 2026 - 07:25:32 | INFO | fastgen.callbacks.wandb:on_validation_end:683 ] Logged 10 val videos at iteration 300 +[Apr 20, 2026 - 07:26:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122737 43.235138 42.991848 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 71.490723 72.750000 70.191406 +util 98.750000 99.000000 98.000000 +[Apr 20, 2026 - 07:26:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 301 : avg iteration time 164.59 seconds +[Apr 20, 2026 - 07:26:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 301 : data loading time 0.00 +[Apr 20, 2026 - 07:26:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 301 : avg forward pass time 10.68 +[Apr 20, 2026 - 07:26:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 301 : backward pass time 12.91 +[Apr 20, 2026 - 07:26:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 301 : optimizer step time 1.12 +[Apr 20, 2026 - 07:26:19 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 301-------------------- +[Apr 20, 2026 - 07:26:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0104 iter count: 1.0 +[Apr 20, 2026 - 07:26:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[Apr 20, 2026 - 07:26:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 07:26:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 07:27:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122802 43.235397 42.991848 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 71.476074 72.789062 70.074219 +util 97.000000 99.000000 94.000000 +[Apr 20, 2026 - 07:27:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 302 : avg iteration time 54.47 seconds +[Apr 20, 2026 - 07:27:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 302 : data loading time 0.00 +[Apr 20, 2026 - 07:27:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 302 : avg forward pass time 15.18 +[Apr 20, 2026 - 07:27:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 302 : backward pass time 11.57 +[Apr 20, 2026 - 07:27:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 302 : optimizer step time 0.98 +[Apr 20, 2026 - 07:27:14 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 302-------------------- +[Apr 20, 2026 - 07:27:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0078 iter count: 1.0 +[Apr 20, 2026 - 07:27:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0078 iter count: 1.0 +[Apr 20, 2026 - 07:27:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 07:27:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 07:28:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122805 43.235409 42.991848 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 71.461426 72.808594 70.015625 +util 97.500000 100.000000 93.000000 +[Apr 20, 2026 - 07:28:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 303 : avg iteration time 50.62 seconds +[Apr 20, 2026 - 07:28:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 303 : data loading time 0.00 +[Apr 20, 2026 - 07:28:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 303 : avg forward pass time 12.84 +[Apr 20, 2026 - 07:28:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 303 : backward pass time 11.78 +[Apr 20, 2026 - 07:28:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 303 : optimizer step time 1.51 +[Apr 20, 2026 - 07:28:04 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 303-------------------- +[Apr 20, 2026 - 07:28:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0085 iter count: 1.0 +[Apr 20, 2026 - 07:28:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[Apr 20, 2026 - 07:28:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 07:28:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 07:28:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122062 43.232437 42.991848 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 71.456543 72.750000 70.074219 +util 98.750000 100.000000 97.000000 +[Apr 20, 2026 - 07:28:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 304 : avg iteration time 47.70 seconds +[Apr 20, 2026 - 07:28:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 304 : data loading time 0.00 +[Apr 20, 2026 - 07:28:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 304 : avg forward pass time 11.15 +[Apr 20, 2026 - 07:28:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 304 : backward pass time 12.77 +[Apr 20, 2026 - 07:28:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 304 : optimizer step time 1.06 +[Apr 20, 2026 - 07:28:52 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 304-------------------- +[Apr 20, 2026 - 07:28:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0092 iter count: 1.0 +[Apr 20, 2026 - 07:28:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[Apr 20, 2026 - 07:28:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 07:28:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 07:31:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122581 43.233162 42.992298 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.502930 135.517578 135.458984 +util 70.750000 77.000000 61.000000 +[Apr 20, 2026 - 07:31:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 305 : avg iteration time 181.41 seconds +[Apr 20, 2026 - 07:31:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 305 : data loading time 0.00 +[Apr 20, 2026 - 07:31:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 305 : avg forward pass time 82.22 +[Apr 20, 2026 - 07:31:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 305 : backward pass time 8.35 +[Apr 20, 2026 - 07:31:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 305 : optimizer step time 0.33 +[Apr 20, 2026 - 07:31:53 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 305-------------------- +[Apr 20, 2026 - 07:31:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 73530.9180 iter count: 1.0 +[Apr 20, 2026 - 07:31:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6679 iter count: 1.0 +[Apr 20, 2026 - 07:31:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 73530.9180iter count: 1.0 +[Apr 20, 2026 - 07:31:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 07:31:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.2767iter count: 1.0 +[Apr 20, 2026 - 07:31:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.3328 iter count: 1.0 +[Apr 20, 2026 - 07:31:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.3672 iter count: 1.0 +[Apr 20, 2026 - 07:31:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 114964.7969iter count: 1.0 +[Apr 20, 2026 - 07:31:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 14.3772iter count: 1.0 +[Apr 20, 2026 - 07:31:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 2506511.7500iter count: 1.0 +[Apr 20, 2026 - 07:31:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6679iter count: 1.0 +[Apr 20, 2026 - 07:31:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 5.2552 iter count: 1.0 +[Apr 20, 2026 - 07:31:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 36694.4141iter count: 1.0 +[Apr 20, 2026 - 07:31:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.8486 iter count: 1.0 +[Apr 20, 2026 - 07:31:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 2201.9900iter count: 1.0 +[Apr 20, 2026 - 07:31:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 5.8920 iter count: 1.0 +[Apr 20, 2026 - 07:31:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 131148.5938iter count: 1.0 +[Apr 20, 2026 - 07:31:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 2.7040 iter count: 1.0 +[Apr 20, 2026 - 07:31:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 223.2037iter count: 1.0 +[Apr 20, 2026 - 07:31:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0122 iter count: 1.0 +[Apr 20, 2026 - 07:31:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 07:32:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122586 43.233185 42.992298 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.918457 69.976562 69.783203 +util 99.000000 100.000000 97.000000 +[Apr 20, 2026 - 07:32:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 306 : avg iteration time 60.58 seconds +[Apr 20, 2026 - 07:32:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 306 : data loading time 0.00 +[Apr 20, 2026 - 07:32:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 306 : avg forward pass time 17.65 +[Apr 20, 2026 - 07:32:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 306 : backward pass time 12.71 +[Apr 20, 2026 - 07:32:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 306 : optimizer step time 1.04 +[Apr 20, 2026 - 07:32:54 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 306-------------------- +[Apr 20, 2026 - 07:32:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0115 iter count: 1.0 +[Apr 20, 2026 - 07:32:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[Apr 20, 2026 - 07:32:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 07:32:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 07:33:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122612 43.233276 42.992310 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.889160 70.035156 69.646484 +util 96.000000 100.000000 91.000000 +[Apr 20, 2026 - 07:33:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 307 : avg iteration time 49.83 seconds +[Apr 20, 2026 - 07:33:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 307 : data loading time 0.00 +[Apr 20, 2026 - 07:33:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 307 : avg forward pass time 12.88 +[Apr 20, 2026 - 07:33:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 307 : backward pass time 11.54 +[Apr 20, 2026 - 07:33:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 307 : optimizer step time 0.99 +[Apr 20, 2026 - 07:33:44 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 307-------------------- +[Apr 20, 2026 - 07:33:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0089 iter count: 1.0 +[Apr 20, 2026 - 07:33:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[Apr 20, 2026 - 07:33:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 07:33:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 07:34:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122608 43.233253 42.992298 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.801270 69.937500 69.490234 +util 96.750000 100.000000 90.000000 +[Apr 20, 2026 - 07:34:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 308 : avg iteration time 52.97 seconds +[Apr 20, 2026 - 07:34:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 308 : data loading time 0.00 +[Apr 20, 2026 - 07:34:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 308 : avg forward pass time 13.79 +[Apr 20, 2026 - 07:34:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 308 : backward pass time 11.54 +[Apr 20, 2026 - 07:34:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 308 : optimizer step time 0.98 +[Apr 20, 2026 - 07:34:37 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 308-------------------- +[Apr 20, 2026 - 07:34:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0084 iter count: 1.0 +[Apr 20, 2026 - 07:34:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[Apr 20, 2026 - 07:34:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 07:34:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 07:35:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122652 43.233448 42.992298 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.708496 69.839844 69.353516 +util 98.750000 100.000000 95.000000 +[Apr 20, 2026 - 07:35:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 309 : avg iteration time 51.39 seconds +[Apr 20, 2026 - 07:35:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 309 : data loading time 0.00 +[Apr 20, 2026 - 07:35:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 309 : avg forward pass time 12.86 +[Apr 20, 2026 - 07:35:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 309 : backward pass time 12.76 +[Apr 20, 2026 - 07:35:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 309 : optimizer step time 1.35 +[Apr 20, 2026 - 07:35:28 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 309-------------------- +[Apr 20, 2026 - 07:35:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0085 iter count: 1.0 +[Apr 20, 2026 - 07:35:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[Apr 20, 2026 - 07:35:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 07:35:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 07:38:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122403 43.232452 42.992298 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.468262 135.478516 135.441406 +util 70.750000 77.000000 65.000000 +[Apr 20, 2026 - 07:38:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 310 : avg iteration time 190.64 seconds +[Apr 20, 2026 - 07:38:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 310 : data loading time 0.00 +[Apr 20, 2026 - 07:38:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 310 : avg forward pass time 86.83 +[Apr 20, 2026 - 07:38:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 310 : backward pass time 8.34 +[Apr 20, 2026 - 07:38:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 310 : optimizer step time 0.33 +[Apr 20, 2026 - 07:38:39 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 310-------------------- +[Apr 20, 2026 - 07:38:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 681783.8784 iter count: 1.0 +[Apr 20, 2026 - 07:38:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6489 iter count: 1.0 +[Apr 20, 2026 - 07:38:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 681783.8784iter count: 1.0 +[Apr 20, 2026 - 07:38:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 07:38:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.8631iter count: 1.0 +[Apr 20, 2026 - 07:38:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 2.5863 iter count: 1.0 +[Apr 20, 2026 - 07:38:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 8.6672 iter count: 1.0 +[Apr 20, 2026 - 07:38:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 1303514.3750iter count: 1.0 +[Apr 20, 2026 - 07:38:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 176.3751iter count: 1.0 +[Apr 20, 2026 - 07:38:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 33747644.0000iter count: 1.0 +[Apr 20, 2026 - 07:38:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6489iter count: 1.0 +[Apr 20, 2026 - 07:38:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 5.0312 iter count: 1.0 +[Apr 20, 2026 - 07:38:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 23446.5586iter count: 1.0 +[Apr 20, 2026 - 07:38:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 7.4560 iter count: 1.0 +[Apr 20, 2026 - 07:38:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 2993740.2500iter count: 1.0 +[Apr 20, 2026 - 07:38:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 6.7573 iter count: 1.0 +[Apr 20, 2026 - 07:38:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 740162.9375iter count: 1.0 +[Apr 20, 2026 - 07:38:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 5.2793 iter count: 1.0 +[Apr 20, 2026 - 07:38:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 38509.8047iter count: 1.0 +[Apr 20, 2026 - 07:38:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[Apr 20, 2026 - 07:38:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 07:39:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122477 43.232742 42.992302 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.947754 70.015625 69.761719 +util 98.250000 100.000000 95.000000 +[Apr 20, 2026 - 07:39:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 311 : avg iteration time 55.32 seconds +[Apr 20, 2026 - 07:39:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 311 : data loading time 0.00 +[Apr 20, 2026 - 07:39:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 311 : avg forward pass time 14.91 +[Apr 20, 2026 - 07:39:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 311 : backward pass time 12.87 +[Apr 20, 2026 - 07:39:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 311 : optimizer step time 1.08 +[Apr 20, 2026 - 07:39:34 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 311-------------------- +[Apr 20, 2026 - 07:39:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0090 iter count: 1.0 +[Apr 20, 2026 - 07:39:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[Apr 20, 2026 - 07:39:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 07:39:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 07:40:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122232 43.231770 42.992298 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.830566 70.054688 69.587891 +util 98.750000 100.000000 96.000000 +[Apr 20, 2026 - 07:40:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 312 : avg iteration time 60.42 seconds +[Apr 20, 2026 - 07:40:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 312 : data loading time 0.00 +[Apr 20, 2026 - 07:40:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 312 : avg forward pass time 18.04 +[Apr 20, 2026 - 07:40:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 312 : backward pass time 11.57 +[Apr 20, 2026 - 07:40:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 312 : optimizer step time 0.99 +[Apr 20, 2026 - 07:40:35 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 312-------------------- +[Apr 20, 2026 - 07:40:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0086 iter count: 1.0 +[Apr 20, 2026 - 07:40:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[Apr 20, 2026 - 07:40:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 07:40:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 07:41:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122263 43.231853 42.992321 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.801270 69.937500 69.548828 +util 98.000000 100.000000 94.000000 +[Apr 20, 2026 - 07:41:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 313 : avg iteration time 56.77 seconds +[Apr 20, 2026 - 07:41:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 313 : data loading time 0.00 +[Apr 20, 2026 - 07:41:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 313 : avg forward pass time 15.58 +[Apr 20, 2026 - 07:41:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 313 : backward pass time 12.86 +[Apr 20, 2026 - 07:41:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 313 : optimizer step time 1.10 +[Apr 20, 2026 - 07:41:31 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 313-------------------- +[Apr 20, 2026 - 07:41:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0112 iter count: 1.0 +[Apr 20, 2026 - 07:41:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[Apr 20, 2026 - 07:41:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 07:41:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 07:42:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122250 43.231838 42.992298 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.884277 70.074219 69.626953 +util 96.750000 99.000000 92.000000 +[Apr 20, 2026 - 07:42:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 314 : avg iteration time 55.17 seconds +[Apr 20, 2026 - 07:42:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 314 : data loading time 0.00 +[Apr 20, 2026 - 07:42:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 314 : avg forward pass time 15.53 +[Apr 20, 2026 - 07:42:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 314 : backward pass time 11.56 +[Apr 20, 2026 - 07:42:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 314 : optimizer step time 0.98 +[Apr 20, 2026 - 07:42:27 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 314-------------------- +[Apr 20, 2026 - 07:42:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0119 iter count: 1.0 +[Apr 20, 2026 - 07:42:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[Apr 20, 2026 - 07:42:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 07:42:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 07:45:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122096 43.231220 42.992302 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.507324 135.535156 135.458984 +util 76.500000 95.000000 64.000000 +[Apr 20, 2026 - 07:45:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 315 : avg iteration time 177.68 seconds +[Apr 20, 2026 - 07:45:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 315 : data loading time 0.00 +[Apr 20, 2026 - 07:45:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 315 : avg forward pass time 79.93 +[Apr 20, 2026 - 07:45:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 315 : backward pass time 9.19 +[Apr 20, 2026 - 07:45:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 315 : optimizer step time 0.33 +[Apr 20, 2026 - 07:45:24 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 315-------------------- +[Apr 20, 2026 - 07:45:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 639970.0303 iter count: 1.0 +[Apr 20, 2026 - 07:45:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6055 iter count: 1.0 +[Apr 20, 2026 - 07:45:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 639970.0303iter count: 1.0 +[Apr 20, 2026 - 07:45:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 07:45:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.8315iter count: 1.0 +[Apr 20, 2026 - 07:45:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.0795 iter count: 1.0 +[Apr 20, 2026 - 07:45:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 8.4532 iter count: 1.0 +[Apr 20, 2026 - 07:45:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 800129.8125iter count: 1.0 +[Apr 20, 2026 - 07:45:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 8.6631 iter count: 1.0 +[Apr 20, 2026 - 07:45:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 21996102.0000iter count: 1.0 +[Apr 20, 2026 - 07:45:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6055iter count: 1.0 +[Apr 20, 2026 - 07:45:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 1.1352 iter count: 1.0 +[Apr 20, 2026 - 07:45:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 9.6838 iter count: 1.0 +[Apr 20, 2026 - 07:45:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.0658 iter count: 1.0 +[Apr 20, 2026 - 07:45:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 3400.4155iter count: 1.0 +[Apr 20, 2026 - 07:45:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 5.3958 iter count: 1.0 +[Apr 20, 2026 - 07:45:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 48614.8867iter count: 1.0 +[Apr 20, 2026 - 07:45:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 5.2187 iter count: 1.0 +[Apr 20, 2026 - 07:45:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 34110.8984iter count: 1.0 +[Apr 20, 2026 - 07:45:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[Apr 20, 2026 - 07:45:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 07:46:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122089 43.231197 42.992298 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 71.314941 72.906250 69.587891 +util 96.250000 99.000000 91.000000 +[Apr 20, 2026 - 07:46:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 316 : avg iteration time 62.71 seconds +[Apr 20, 2026 - 07:46:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 316 : data loading time 0.00 +[Apr 20, 2026 - 07:46:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 316 : avg forward pass time 19.29 +[Apr 20, 2026 - 07:46:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 316 : backward pass time 11.57 +[Apr 20, 2026 - 07:46:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 316 : optimizer step time 0.97 +[Apr 20, 2026 - 07:46:27 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 316-------------------- +[Apr 20, 2026 - 07:46:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0101 iter count: 1.0 +[Apr 20, 2026 - 07:46:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[Apr 20, 2026 - 07:46:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 07:46:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 07:47:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122293 43.232014 42.992298 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.880371 70.017578 69.587891 +util 98.000000 100.000000 93.000000 +[Apr 20, 2026 - 07:47:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 317 : avg iteration time 59.31 seconds +[Apr 20, 2026 - 07:47:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 317 : data loading time 0.00 +[Apr 20, 2026 - 07:47:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 317 : avg forward pass time 16.95 +[Apr 20, 2026 - 07:47:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 317 : backward pass time 12.79 +[Apr 20, 2026 - 07:47:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 317 : optimizer step time 1.07 +[Apr 20, 2026 - 07:47:26 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 317-------------------- +[Apr 20, 2026 - 07:47:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0088 iter count: 1.0 +[Apr 20, 2026 - 07:47:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[Apr 20, 2026 - 07:47:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 07:47:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 07:48:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122305 43.232037 42.992298 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.987793 70.017578 69.898438 +util 96.000000 100.000000 92.000000 +[Apr 20, 2026 - 07:48:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 318 : avg iteration time 58.64 seconds +[Apr 20, 2026 - 07:48:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 318 : data loading time 0.00 +[Apr 20, 2026 - 07:48:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 318 : avg forward pass time 17.14 +[Apr 20, 2026 - 07:48:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 318 : backward pass time 11.54 +[Apr 20, 2026 - 07:48:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 318 : optimizer step time 1.00 +[Apr 20, 2026 - 07:48:25 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 318-------------------- +[Apr 20, 2026 - 07:48:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0089 iter count: 1.0 +[Apr 20, 2026 - 07:48:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[Apr 20, 2026 - 07:48:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 07:48:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 07:49:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122305 43.232037 42.992298 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.904785 70.037109 69.646484 +util 98.250000 100.000000 93.000000 +[Apr 20, 2026 - 07:49:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 319 : avg iteration time 64.56 seconds +[Apr 20, 2026 - 07:49:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 319 : data loading time 0.00 +[Apr 20, 2026 - 07:49:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 319 : avg forward pass time 19.25 +[Apr 20, 2026 - 07:49:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 319 : backward pass time 11.65 +[Apr 20, 2026 - 07:49:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 319 : optimizer step time 1.42 +[Apr 20, 2026 - 07:49:29 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 319-------------------- +[Apr 20, 2026 - 07:49:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0118 iter count: 1.0 +[Apr 20, 2026 - 07:49:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0118 iter count: 1.0 +[Apr 20, 2026 - 07:49:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 07:49:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 07:52:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122015 43.230896 42.992298 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.468262 135.478516 135.457031 +util 70.000000 78.000000 64.000000 +[Apr 20, 2026 - 07:52:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 320 : avg iteration time 187.01 seconds +[Apr 20, 2026 - 07:52:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 320 : data loading time 0.00 +[Apr 20, 2026 - 07:52:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 320 : avg forward pass time 84.95 +[Apr 20, 2026 - 07:52:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 320 : backward pass time 8.42 +[Apr 20, 2026 - 07:52:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 320 : optimizer step time 0.33 +[Apr 20, 2026 - 07:52:36 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 320-------------------- +[Apr 20, 2026 - 07:52:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 61815.3423 iter count: 1.0 +[Apr 20, 2026 - 07:52:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6184 iter count: 1.0 +[Apr 20, 2026 - 07:52:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 61815.3423iter count: 1.0 +[Apr 20, 2026 - 07:52:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 07:52:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.4237iter count: 1.0 +[Apr 20, 2026 - 07:52:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.4975 iter count: 1.0 +[Apr 20, 2026 - 07:52:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.8736 iter count: 1.0 +[Apr 20, 2026 - 07:52:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 87055.0938iter count: 1.0 +[Apr 20, 2026 - 07:52:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 2.7050 iter count: 1.0 +[Apr 20, 2026 - 07:52:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 934044.3125iter count: 1.0 +[Apr 20, 2026 - 07:52:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6184iter count: 1.0 +[Apr 20, 2026 - 07:52:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 0.4975 iter count: 1.0 +[Apr 20, 2026 - 07:52:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 2.7050 iter count: 1.0 +[Apr 20, 2026 - 07:52:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.8071 iter count: 1.0 +[Apr 20, 2026 - 07:52:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 14975.1465iter count: 1.0 +[Apr 20, 2026 - 07:52:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 3.4305 iter count: 1.0 +[Apr 20, 2026 - 07:52:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 954.3337iter count: 1.0 +[Apr 20, 2026 - 07:52:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 5.4238 iter count: 1.0 +[Apr 20, 2026 - 07:52:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 51406.2539iter count: 1.0 +[Apr 20, 2026 - 07:52:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[Apr 20, 2026 - 07:52:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 07:53:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122103 43.231251 42.992298 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.864746 70.017578 69.800781 +util 97.750000 100.000000 92.000000 +[Apr 20, 2026 - 07:53:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 321 : avg iteration time 61.18 seconds +[Apr 20, 2026 - 07:53:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 321 : data loading time 0.00 +[Apr 20, 2026 - 07:53:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 321 : avg forward pass time 17.45 +[Apr 20, 2026 - 07:53:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 321 : backward pass time 12.45 +[Apr 20, 2026 - 07:53:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 321 : optimizer step time 1.58 +[Apr 20, 2026 - 07:53:38 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 321-------------------- +[Apr 20, 2026 - 07:53:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0132 iter count: 1.0 +[Apr 20, 2026 - 07:53:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0132 iter count: 1.0 +[Apr 20, 2026 - 07:53:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 07:53:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 07:54:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122108 43.231251 42.992321 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.942871 70.037109 69.820312 +util 96.250000 99.000000 91.000000 +[Apr 20, 2026 - 07:54:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 322 : avg iteration time 60.89 seconds +[Apr 20, 2026 - 07:54:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 322 : data loading time 0.00 +[Apr 20, 2026 - 07:54:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 322 : avg forward pass time 18.40 +[Apr 20, 2026 - 07:54:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 322 : backward pass time 11.56 +[Apr 20, 2026 - 07:54:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 322 : optimizer step time 0.97 +[Apr 20, 2026 - 07:54:39 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 322-------------------- +[Apr 20, 2026 - 07:54:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0109 iter count: 1.0 +[Apr 20, 2026 - 07:54:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[Apr 20, 2026 - 07:54:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 07:54:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 07:55:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122108 43.231274 42.992298 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.811035 69.917969 69.607422 +util 97.750000 100.000000 92.000000 +[Apr 20, 2026 - 07:55:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 323 : avg iteration time 47.71 seconds +[Apr 20, 2026 - 07:55:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 323 : data loading time 0.00 +[Apr 20, 2026 - 07:55:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 323 : avg forward pass time 11.09 +[Apr 20, 2026 - 07:55:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 323 : backward pass time 11.56 +[Apr 20, 2026 - 07:55:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 323 : optimizer step time 0.98 +[Apr 20, 2026 - 07:55:26 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 323-------------------- +[Apr 20, 2026 - 07:55:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0107 iter count: 1.0 +[Apr 20, 2026 - 07:55:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[Apr 20, 2026 - 07:55:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 07:55:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 07:56:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122108 43.231251 42.992317 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.947754 70.037109 69.800781 +util 96.750000 100.000000 93.000000 +[Apr 20, 2026 - 07:56:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 324 : avg iteration time 59.66 seconds +[Apr 20, 2026 - 07:56:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 324 : data loading time 0.00 +[Apr 20, 2026 - 07:56:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 324 : avg forward pass time 17.01 +[Apr 20, 2026 - 07:56:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 324 : backward pass time 12.96 +[Apr 20, 2026 - 07:56:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 324 : optimizer step time 1.12 +[Apr 20, 2026 - 07:56:26 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 324-------------------- +[Apr 20, 2026 - 07:56:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0109 iter count: 1.0 +[Apr 20, 2026 - 07:56:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[Apr 20, 2026 - 07:56:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 07:56:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 07:59:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121626 43.229908 42.992298 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.481934 135.515625 135.458984 +util 69.000000 75.000000 60.000000 +[Apr 20, 2026 - 07:59:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 325 : avg iteration time 181.92 seconds +[Apr 20, 2026 - 07:59:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 325 : data loading time 0.00 +[Apr 20, 2026 - 07:59:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 325 : avg forward pass time 82.43 +[Apr 20, 2026 - 07:59:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 325 : backward pass time 8.38 +[Apr 20, 2026 - 07:59:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 325 : optimizer step time 0.35 +[Apr 20, 2026 - 07:59:28 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 325-------------------- +[Apr 20, 2026 - 07:59:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 136667.3416 iter count: 1.0 +[Apr 20, 2026 - 07:59:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6793 iter count: 1.0 +[Apr 20, 2026 - 07:59:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 136667.3416iter count: 1.0 +[Apr 20, 2026 - 07:59:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 07:59:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.1658iter count: 1.0 +[Apr 20, 2026 - 07:59:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.9183 iter count: 1.0 +[Apr 20, 2026 - 07:59:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.5989 iter count: 1.0 +[Apr 20, 2026 - 07:59:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 182106.6094iter count: 1.0 +[Apr 20, 2026 - 07:59:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 6.2747 iter count: 1.0 +[Apr 20, 2026 - 07:59:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 3984288.5000iter count: 1.0 +[Apr 20, 2026 - 07:59:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6793iter count: 1.0 +[Apr 20, 2026 - 07:59:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 1.8434 iter count: 1.0 +[Apr 20, 2026 - 07:59:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 39.9207 iter count: 1.0 +[Apr 20, 2026 - 07:59:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.9334 iter count: 1.0 +[Apr 20, 2026 - 07:59:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 2609.0544iter count: 1.0 +[Apr 20, 2026 - 07:59:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.1516 iter count: 1.0 +[Apr 20, 2026 - 07:59:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 4036.7842iter count: 1.0 +[Apr 20, 2026 - 07:59:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.5372 iter count: 1.0 +[Apr 20, 2026 - 07:59:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 8729.4961iter count: 1.0 +[Apr 20, 2026 - 07:59:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[Apr 20, 2026 - 07:59:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 08:00:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121739 43.230350 42.992306 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.830566 69.878906 69.800781 +util 99.000000 100.000000 98.000000 +[Apr 20, 2026 - 08:00:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 326 : avg iteration time 49.00 seconds +[Apr 20, 2026 - 08:00:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 326 : data loading time 0.00 +[Apr 20, 2026 - 08:00:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 326 : avg forward pass time 11.95 +[Apr 20, 2026 - 08:00:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 326 : backward pass time 11.54 +[Apr 20, 2026 - 08:00:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 326 : optimizer step time 0.98 +[Apr 20, 2026 - 08:00:17 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 326-------------------- +[Apr 20, 2026 - 08:00:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0115 iter count: 1.0 +[Apr 20, 2026 - 08:00:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[Apr 20, 2026 - 08:00:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 08:00:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 08:01:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121932 43.231102 42.992321 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.815918 69.898438 69.568359 +util 96.750000 100.000000 92.000000 +[Apr 20, 2026 - 08:01:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 327 : avg iteration time 51.73 seconds +[Apr 20, 2026 - 08:01:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 327 : data loading time 0.00 +[Apr 20, 2026 - 08:01:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 327 : avg forward pass time 12.97 +[Apr 20, 2026 - 08:01:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 327 : backward pass time 12.81 +[Apr 20, 2026 - 08:01:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 327 : optimizer step time 1.43 +[Apr 20, 2026 - 08:01:09 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 327-------------------- +[Apr 20, 2026 - 08:01:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0095 iter count: 1.0 +[Apr 20, 2026 - 08:01:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[Apr 20, 2026 - 08:01:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 08:01:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 08:02:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121922 43.231079 42.992298 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.840332 69.976562 69.548828 +util 95.000000 99.000000 91.000000 +[Apr 20, 2026 - 08:02:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 328 : avg iteration time 52.94 seconds +[Apr 20, 2026 - 08:02:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 328 : data loading time 0.00 +[Apr 20, 2026 - 08:02:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 328 : avg forward pass time 14.40 +[Apr 20, 2026 - 08:02:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 328 : backward pass time 11.54 +[Apr 20, 2026 - 08:02:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 328 : optimizer step time 1.02 +[Apr 20, 2026 - 08:02:01 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 328-------------------- +[Apr 20, 2026 - 08:02:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0094 iter count: 1.0 +[Apr 20, 2026 - 08:02:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[Apr 20, 2026 - 08:02:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 08:02:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 08:03:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121919 43.231079 42.992298 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.854980 70.015625 69.568359 +util 98.750000 100.000000 97.000000 +[Apr 20, 2026 - 08:03:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 329 : avg iteration time 58.52 seconds +[Apr 20, 2026 - 08:03:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 329 : data loading time 0.00 +[Apr 20, 2026 - 08:03:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 329 : avg forward pass time 16.49 +[Apr 20, 2026 - 08:03:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 329 : backward pass time 11.55 +[Apr 20, 2026 - 08:03:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 329 : optimizer step time 1.02 +[Apr 20, 2026 - 08:03:00 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 329-------------------- +[Apr 20, 2026 - 08:03:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0089 iter count: 1.0 +[Apr 20, 2026 - 08:03:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[Apr 20, 2026 - 08:03:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 08:03:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 08:06:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121799 43.230591 42.992298 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.463379 135.519531 135.417969 +util 71.500000 75.000000 66.000000 +[Apr 20, 2026 - 08:06:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 330 : avg iteration time 193.79 seconds +[Apr 20, 2026 - 08:06:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 330 : data loading time 0.00 +[Apr 20, 2026 - 08:06:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 330 : avg forward pass time 88.39 +[Apr 20, 2026 - 08:06:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 330 : backward pass time 8.34 +[Apr 20, 2026 - 08:06:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 330 : optimizer step time 0.33 +[Apr 20, 2026 - 08:06:14 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 330-------------------- +[Apr 20, 2026 - 08:06:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 9987.5453 iter count: 1.0 +[Apr 20, 2026 - 08:06:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6242 iter count: 1.0 +[Apr 20, 2026 - 08:06:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 9987.5453iter count: 1.0 +[Apr 20, 2026 - 08:06:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 08:06:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.8627iter count: 1.0 +[Apr 20, 2026 - 08:06:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.1679 iter count: 1.0 +[Apr 20, 2026 - 08:06:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 5.7194 iter count: 1.0 +[Apr 20, 2026 - 08:06:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 16188.9004iter count: 1.0 +[Apr 20, 2026 - 08:06:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 10.3382iter count: 1.0 +[Apr 20, 2026 - 08:06:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 92857.7188iter count: 1.0 +[Apr 20, 2026 - 08:06:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6242iter count: 1.0 +[Apr 20, 2026 - 08:06:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.1259 iter count: 1.0 +[Apr 20, 2026 - 08:06:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 3834.8311iter count: 1.0 +[Apr 20, 2026 - 08:06:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 1.3252 iter count: 1.0 +[Apr 20, 2026 - 08:06:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 14.1608 iter count: 1.0 +[Apr 20, 2026 - 08:06:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 2.2183 iter count: 1.0 +[Apr 20, 2026 - 08:06:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 84.4891 iter count: 1.0 +[Apr 20, 2026 - 08:06:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 2.4993 iter count: 1.0 +[Apr 20, 2026 - 08:06:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 148.1985iter count: 1.0 +[Apr 20, 2026 - 08:06:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[Apr 20, 2026 - 08:06:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 08:07:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121861 43.230839 42.992298 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.845215 69.957031 69.568359 +util 96.500000 99.000000 93.000000 +[Apr 20, 2026 - 08:07:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 331 : avg iteration time 58.32 seconds +[Apr 20, 2026 - 08:07:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 331 : data loading time 0.00 +[Apr 20, 2026 - 08:07:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 331 : avg forward pass time 16.33 +[Apr 20, 2026 - 08:07:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 331 : backward pass time 12.96 +[Apr 20, 2026 - 08:07:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 331 : optimizer step time 1.13 +[Apr 20, 2026 - 08:07:12 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 331-------------------- +[Apr 20, 2026 - 08:07:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0106 iter count: 1.0 +[Apr 20, 2026 - 08:07:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[Apr 20, 2026 - 08:07:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 08:07:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 08:08:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121873 43.230873 42.992321 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.825684 69.976562 69.490234 +util 95.250000 100.000000 91.000000 +[Apr 20, 2026 - 08:08:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 332 : avg iteration time 57.98 seconds +[Apr 20, 2026 - 08:08:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 332 : data loading time 0.00 +[Apr 20, 2026 - 08:08:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 332 : avg forward pass time 16.93 +[Apr 20, 2026 - 08:08:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 332 : backward pass time 11.55 +[Apr 20, 2026 - 08:08:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 332 : optimizer step time 0.99 +[Apr 20, 2026 - 08:08:10 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 332-------------------- +[Apr 20, 2026 - 08:08:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0116 iter count: 1.0 +[Apr 20, 2026 - 08:08:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[Apr 20, 2026 - 08:08:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 08:08:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 08:09:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121922 43.231068 42.992298 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.825684 69.976562 69.529297 +util 98.750000 100.000000 95.000000 +[Apr 20, 2026 - 08:09:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 333 : avg iteration time 56.50 seconds +[Apr 20, 2026 - 08:09:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 333 : data loading time 0.00 +[Apr 20, 2026 - 08:09:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 333 : avg forward pass time 15.16 +[Apr 20, 2026 - 08:09:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 333 : backward pass time 11.81 +[Apr 20, 2026 - 08:09:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 333 : optimizer step time 1.50 +[Apr 20, 2026 - 08:09:07 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 333-------------------- +[Apr 20, 2026 - 08:09:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0117 iter count: 1.0 +[Apr 20, 2026 - 08:09:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[Apr 20, 2026 - 08:09:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 08:09:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 08:09:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121943 43.231171 42.992302 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.820801 69.996094 69.509766 +util 98.000000 100.000000 94.000000 +[Apr 20, 2026 - 08:09:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 334 : avg iteration time 52.59 seconds +[Apr 20, 2026 - 08:09:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 334 : data loading time 0.00 +[Apr 20, 2026 - 08:09:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 334 : avg forward pass time 14.18 +[Apr 20, 2026 - 08:09:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 334 : backward pass time 11.67 +[Apr 20, 2026 - 08:09:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 334 : optimizer step time 0.98 +[Apr 20, 2026 - 08:09:59 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 334-------------------- +[Apr 20, 2026 - 08:09:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0085 iter count: 1.0 +[Apr 20, 2026 - 08:09:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[Apr 20, 2026 - 08:09:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 08:09:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 08:13:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121870 43.230831 42.992302 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.492676 135.535156 135.439453 +util 67.750000 75.000000 63.000000 +[Apr 20, 2026 - 08:13:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 335 : avg iteration time 182.72 seconds +[Apr 20, 2026 - 08:13:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 335 : data loading time 0.00 +[Apr 20, 2026 - 08:13:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 335 : avg forward pass time 82.87 +[Apr 20, 2026 - 08:13:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 335 : backward pass time 8.35 +[Apr 20, 2026 - 08:13:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 335 : optimizer step time 0.33 +[Apr 20, 2026 - 08:13:02 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 335-------------------- +[Apr 20, 2026 - 08:13:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 280792.6758 iter count: 1.0 +[Apr 20, 2026 - 08:13:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6423 iter count: 1.0 +[Apr 20, 2026 - 08:13:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 280792.6758iter count: 1.0 +[Apr 20, 2026 - 08:13:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 08:13:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.4023iter count: 1.0 +[Apr 20, 2026 - 08:13:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.5463 iter count: 1.0 +[Apr 20, 2026 - 08:13:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.9185 iter count: 1.0 +[Apr 20, 2026 - 08:13:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 370815.7500iter count: 1.0 +[Apr 20, 2026 - 08:13:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 2.9818 iter count: 1.0 +[Apr 20, 2026 - 08:13:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 7549567.5000iter count: 1.0 +[Apr 20, 2026 - 08:13:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6423iter count: 1.0 +[Apr 20, 2026 - 08:13:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 5.9823 iter count: 1.0 +[Apr 20, 2026 - 08:13:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 157078.4219iter count: 1.0 +[Apr 20, 2026 - 08:13:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.1278 iter count: 1.0 +[Apr 20, 2026 - 08:13:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 3848.9373iter count: 1.0 +[Apr 20, 2026 - 08:13:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.1386 iter count: 1.0 +[Apr 20, 2026 - 08:13:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 3932.9141iter count: 1.0 +[Apr 20, 2026 - 08:13:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 5.0299 iter count: 1.0 +[Apr 20, 2026 - 08:13:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 23381.4922iter count: 1.0 +[Apr 20, 2026 - 08:13:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[Apr 20, 2026 - 08:13:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 08:14:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121862 43.230831 42.992317 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.869629 69.937500 69.685547 +util 97.750000 100.000000 93.000000 +[Apr 20, 2026 - 08:14:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 336 : avg iteration time 58.36 seconds +[Apr 20, 2026 - 08:14:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 336 : data loading time 0.00 +[Apr 20, 2026 - 08:14:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 336 : avg forward pass time 16.38 +[Apr 20, 2026 - 08:14:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 336 : backward pass time 12.92 +[Apr 20, 2026 - 08:14:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 336 : optimizer step time 1.14 +[Apr 20, 2026 - 08:14:00 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 336-------------------- +[Apr 20, 2026 - 08:14:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0101 iter count: 1.0 +[Apr 20, 2026 - 08:14:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[Apr 20, 2026 - 08:14:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 08:14:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 08:14:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121986 43.231293 42.992329 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.757324 69.917969 69.294922 +util 98.500000 99.000000 97.000000 +[Apr 20, 2026 - 08:14:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 337 : avg iteration time 49.76 seconds +[Apr 20, 2026 - 08:14:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 337 : data loading time 0.00 +[Apr 20, 2026 - 08:14:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 337 : avg forward pass time 12.82 +[Apr 20, 2026 - 08:14:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 337 : backward pass time 11.53 +[Apr 20, 2026 - 08:14:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 337 : optimizer step time 1.01 +[Apr 20, 2026 - 08:14:50 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 337-------------------- +[Apr 20, 2026 - 08:14:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0095 iter count: 1.0 +[Apr 20, 2026 - 08:14:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[Apr 20, 2026 - 08:14:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 08:14:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 08:15:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122098 43.231766 42.992306 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.903809 69.937500 69.802734 +util 99.000000 100.000000 96.000000 +[Apr 20, 2026 - 08:15:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 338 : avg iteration time 61.59 seconds +[Apr 20, 2026 - 08:15:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 338 : data loading time 0.00 +[Apr 20, 2026 - 08:15:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 338 : avg forward pass time 17.91 +[Apr 20, 2026 - 08:15:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 338 : backward pass time 11.62 +[Apr 20, 2026 - 08:15:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 338 : optimizer step time 1.35 +[Apr 20, 2026 - 08:15:52 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 338-------------------- +[Apr 20, 2026 - 08:15:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0101 iter count: 1.0 +[Apr 20, 2026 - 08:15:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[Apr 20, 2026 - 08:15:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 08:15:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 08:16:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122098 43.231766 42.992306 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.801270 69.937500 69.451172 +util 95.250000 100.000000 91.000000 +[Apr 20, 2026 - 08:16:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 339 : avg iteration time 58.09 seconds +[Apr 20, 2026 - 08:16:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 339 : data loading time 0.00 +[Apr 20, 2026 - 08:16:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 339 : avg forward pass time 17.00 +[Apr 20, 2026 - 08:16:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 339 : backward pass time 11.54 +[Apr 20, 2026 - 08:16:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 339 : optimizer step time 0.99 +[Apr 20, 2026 - 08:16:50 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 339-------------------- +[Apr 20, 2026 - 08:16:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0101 iter count: 1.0 +[Apr 20, 2026 - 08:16:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[Apr 20, 2026 - 08:16:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 08:16:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 08:19:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121880 43.230900 42.992310 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.506836 135.535156 135.421875 +util 66.750000 70.000000 62.000000 +[Apr 20, 2026 - 08:19:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 340 : avg iteration time 186.09 seconds +[Apr 20, 2026 - 08:19:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 340 : data loading time 0.00 +[Apr 20, 2026 - 08:19:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 340 : avg forward pass time 83.80 +[Apr 20, 2026 - 08:19:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 340 : backward pass time 9.80 +[Apr 20, 2026 - 08:19:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 340 : optimizer step time 0.37 +[Apr 20, 2026 - 08:19:56 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 340-------------------- +[Apr 20, 2026 - 08:19:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 55102.2183 iter count: 1.0 +[Apr 20, 2026 - 08:19:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6636 iter count: 1.0 +[Apr 20, 2026 - 08:19:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 55102.2183iter count: 1.0 +[Apr 20, 2026 - 08:19:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 08:19:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.5540iter count: 1.0 +[Apr 20, 2026 - 08:19:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 2.5023 iter count: 1.0 +[Apr 20, 2026 - 08:19:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.2793 iter count: 1.0 +[Apr 20, 2026 - 08:19:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 110624.3984iter count: 1.0 +[Apr 20, 2026 - 08:19:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 149.1009iter count: 1.0 +[Apr 20, 2026 - 08:19:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 2102594.2500iter count: 1.0 +[Apr 20, 2026 - 08:19:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6636iter count: 1.0 +[Apr 20, 2026 - 08:19:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 5.8235 iter count: 1.0 +[Apr 20, 2026 - 08:19:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 114358.7500iter count: 1.0 +[Apr 20, 2026 - 08:19:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 5.2897 iter count: 1.0 +[Apr 20, 2026 - 08:19:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 39319.9336iter count: 1.0 +[Apr 20, 2026 - 08:19:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 3.1091 iter count: 1.0 +[Apr 20, 2026 - 08:19:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 501.8120iter count: 1.0 +[Apr 20, 2026 - 08:19:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 3.6889 iter count: 1.0 +[Apr 20, 2026 - 08:19:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 1600.1816iter count: 1.0 +[Apr 20, 2026 - 08:19:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[Apr 20, 2026 - 08:19:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 08:21:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121886 43.230900 42.992306 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 71.173340 72.710938 69.314453 +util 94.750000 99.000000 91.000000 +[Apr 20, 2026 - 08:21:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 341 : avg iteration time 64.51 seconds +[Apr 20, 2026 - 08:21:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 341 : data loading time 0.00 +[Apr 20, 2026 - 08:21:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 341 : avg forward pass time 20.16 +[Apr 20, 2026 - 08:21:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 341 : backward pass time 11.54 +[Apr 20, 2026 - 08:21:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 341 : optimizer step time 1.00 +[Apr 20, 2026 - 08:21:00 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 341-------------------- +[Apr 20, 2026 - 08:21:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0115 iter count: 1.0 +[Apr 20, 2026 - 08:21:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[Apr 20, 2026 - 08:21:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 08:21:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 08:22:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121884 43.230900 42.992306 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 71.319824 72.789062 69.802734 +util 95.500000 99.000000 92.000000 +[Apr 20, 2026 - 08:22:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 342 : avg iteration time 62.81 seconds +[Apr 20, 2026 - 08:22:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 342 : data loading time 0.00 +[Apr 20, 2026 - 08:22:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 342 : avg forward pass time 18.76 +[Apr 20, 2026 - 08:22:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 342 : backward pass time 12.67 +[Apr 20, 2026 - 08:22:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 342 : optimizer step time 1.04 +[Apr 20, 2026 - 08:22:03 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 342-------------------- +[Apr 20, 2026 - 08:22:03 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0098 iter count: 1.0 +[Apr 20, 2026 - 08:22:03 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[Apr 20, 2026 - 08:22:03 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 08:22:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 08:22:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121939 43.231140 42.992306 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 71.227051 72.750000 69.509766 +util 96.750000 100.000000 93.000000 +[Apr 20, 2026 - 08:22:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 343 : avg iteration time 52.21 seconds +[Apr 20, 2026 - 08:22:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 343 : data loading time 0.00 +[Apr 20, 2026 - 08:22:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 343 : avg forward pass time 14.01 +[Apr 20, 2026 - 08:22:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 343 : backward pass time 11.55 +[Apr 20, 2026 - 08:22:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 343 : optimizer step time 0.98 +[Apr 20, 2026 - 08:22:55 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 343-------------------- +[Apr 20, 2026 - 08:22:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0106 iter count: 1.0 +[Apr 20, 2026 - 08:22:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[Apr 20, 2026 - 08:22:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 08:22:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 08:23:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121944 43.231163 42.992306 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 71.339355 72.789062 69.822266 +util 97.250000 100.000000 91.000000 +[Apr 20, 2026 - 08:23:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 344 : avg iteration time 56.32 seconds +[Apr 20, 2026 - 08:23:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 344 : data loading time 0.00 +[Apr 20, 2026 - 08:23:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 344 : avg forward pass time 15.48 +[Apr 20, 2026 - 08:23:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 344 : backward pass time 12.42 +[Apr 20, 2026 - 08:23:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 344 : optimizer step time 1.41 +[Apr 20, 2026 - 08:23:52 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 344-------------------- +[Apr 20, 2026 - 08:23:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0112 iter count: 1.0 +[Apr 20, 2026 - 08:23:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[Apr 20, 2026 - 08:23:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 08:23:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 08:27:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121696 43.230171 42.992306 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.468750 135.498047 135.458984 +util 70.750000 76.000000 64.000000 +[Apr 20, 2026 - 08:27:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 345 : avg iteration time 190.32 seconds +[Apr 20, 2026 - 08:27:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 345 : data loading time 0.00 +[Apr 20, 2026 - 08:27:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 345 : avg forward pass time 86.10 +[Apr 20, 2026 - 08:27:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 345 : backward pass time 9.50 +[Apr 20, 2026 - 08:27:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 345 : optimizer step time 0.34 +[Apr 20, 2026 - 08:27:02 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 345-------------------- +[Apr 20, 2026 - 08:27:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 243795.9141 iter count: 1.0 +[Apr 20, 2026 - 08:27:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6440 iter count: 1.0 +[Apr 20, 2026 - 08:27:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 243795.9141iter count: 1.0 +[Apr 20, 2026 - 08:27:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 08:27:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.1732iter count: 1.0 +[Apr 20, 2026 - 08:27:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.7966 iter count: 1.0 +[Apr 20, 2026 - 08:27:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.7879 iter count: 1.0 +[Apr 20, 2026 - 08:27:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 315004.8750iter count: 1.0 +[Apr 20, 2026 - 08:27:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 4.9196 iter count: 1.0 +[Apr 20, 2026 - 08:27:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 5814341.5000iter count: 1.0 +[Apr 20, 2026 - 08:27:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6440iter count: 1.0 +[Apr 20, 2026 - 08:27:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 1.1812 iter count: 1.0 +[Apr 20, 2026 - 08:27:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 10.6158 iter count: 1.0 +[Apr 20, 2026 - 08:27:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.9988 iter count: 1.0 +[Apr 20, 2026 - 08:27:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 21972.0840iter count: 1.0 +[Apr 20, 2026 - 08:27:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.0046 iter count: 1.0 +[Apr 20, 2026 - 08:27:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 3008.7996iter count: 1.0 +[Apr 20, 2026 - 08:27:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 1.0015 iter count: 1.0 +[Apr 20, 2026 - 08:27:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 7.4116 iter count: 1.0 +[Apr 20, 2026 - 08:27:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[Apr 20, 2026 - 08:27:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 08:28:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121704 43.230171 42.992306 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 71.241699 72.847656 69.390625 +util 98.750000 100.000000 97.000000 +[Apr 20, 2026 - 08:28:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 346 : avg iteration time 61.86 seconds +[Apr 20, 2026 - 08:28:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 346 : data loading time 0.00 +[Apr 20, 2026 - 08:28:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 346 : avg forward pass time 18.88 +[Apr 20, 2026 - 08:28:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 346 : backward pass time 11.55 +[Apr 20, 2026 - 08:28:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 346 : optimizer step time 0.98 +[Apr 20, 2026 - 08:28:04 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 346-------------------- +[Apr 20, 2026 - 08:28:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0098 iter count: 1.0 +[Apr 20, 2026 - 08:28:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[Apr 20, 2026 - 08:28:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 08:28:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 08:28:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121949 43.231159 42.992306 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 71.241699 72.750000 69.410156 +util 99.500000 100.000000 98.000000 +[Apr 20, 2026 - 08:28:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 347 : avg iteration time 51.01 seconds +[Apr 20, 2026 - 08:28:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 347 : data loading time 0.00 +[Apr 20, 2026 - 08:28:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 347 : avg forward pass time 12.77 +[Apr 20, 2026 - 08:28:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 347 : backward pass time 12.51 +[Apr 20, 2026 - 08:28:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 347 : optimizer step time 1.40 +[Apr 20, 2026 - 08:28:55 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 347-------------------- +[Apr 20, 2026 - 08:28:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0106 iter count: 1.0 +[Apr 20, 2026 - 08:28:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[Apr 20, 2026 - 08:28:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 08:28:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 08:29:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122072 43.231659 42.992310 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 71.134277 72.750000 69.429688 +util 96.000000 99.000000 91.000000 +[Apr 20, 2026 - 08:29:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 348 : avg iteration time 60.92 seconds +[Apr 20, 2026 - 08:29:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 348 : data loading time 0.00 +[Apr 20, 2026 - 08:29:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 348 : avg forward pass time 18.44 +[Apr 20, 2026 - 08:29:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 348 : backward pass time 11.55 +[Apr 20, 2026 - 08:29:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 348 : optimizer step time 0.98 +[Apr 20, 2026 - 08:29:56 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 348-------------------- +[Apr 20, 2026 - 08:29:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0107 iter count: 1.0 +[Apr 20, 2026 - 08:29:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[Apr 20, 2026 - 08:29:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 08:29:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 08:30:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122071 43.231647 42.992306 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 71.124512 72.750000 69.449219 +util 98.500000 100.000000 95.000000 +[Apr 20, 2026 - 08:30:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 349 : avg iteration time 53.86 seconds +[Apr 20, 2026 - 08:30:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 349 : data loading time 0.00 +[Apr 20, 2026 - 08:30:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 349 : avg forward pass time 14.19 +[Apr 20, 2026 - 08:30:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 349 : backward pass time 12.64 +[Apr 20, 2026 - 08:30:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 349 : optimizer step time 1.29 +[Apr 20, 2026 - 08:30:50 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 349-------------------- +[Apr 20, 2026 - 08:30:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0105 iter count: 1.0 +[Apr 20, 2026 - 08:30:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[Apr 20, 2026 - 08:30:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 08:30:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 08:33:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121819 43.230663 42.992306 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.483887 135.537109 135.402344 +util 70.500000 79.000000 63.000000 +[Apr 20, 2026 - 08:33:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 350 : avg iteration time 178.89 seconds +[Apr 20, 2026 - 08:33:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 350 : data loading time 0.00 +[Apr 20, 2026 - 08:33:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 350 : avg forward pass time 80.91 +[Apr 20, 2026 - 08:33:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 350 : backward pass time 8.43 +[Apr 20, 2026 - 08:33:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 350 : optimizer step time 0.33 +[Apr 20, 2026 - 08:33:48 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 350-------------------- +[Apr 20, 2026 - 08:33:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 174349.7654 iter count: 1.0 +[Apr 20, 2026 - 08:33:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6375 iter count: 1.0 +[Apr 20, 2026 - 08:33:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 174349.7654iter count: 1.0 +[Apr 20, 2026 - 08:33:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 08:33:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.3538iter count: 1.0 +[Apr 20, 2026 - 08:33:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.2027 iter count: 1.0 +[Apr 20, 2026 - 08:33:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.6615 iter count: 1.0 +[Apr 20, 2026 - 08:33:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 208338.6719iter count: 1.0 +[Apr 20, 2026 - 08:33:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 11.0836iter count: 1.0 +[Apr 20, 2026 - 08:33:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 4515023.0000iter count: 1.0 +[Apr 20, 2026 - 08:33:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6375iter count: 1.0 +[Apr 20, 2026 - 08:33:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.7894 iter count: 1.0 +[Apr 20, 2026 - 08:33:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 1956.3812iter count: 1.0 +[Apr 20, 2026 - 08:33:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 6.1762 iter count: 1.0 +[Apr 20, 2026 - 08:33:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 231514.9688iter count: 1.0 +[Apr 20, 2026 - 08:33:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 2.2630 iter count: 1.0 +[Apr 20, 2026 - 08:33:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 92.3815 iter count: 1.0 +[Apr 20, 2026 - 08:33:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 6.1798 iter count: 1.0 +[Apr 20, 2026 - 08:33:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 233183.5156iter count: 1.0 +[Apr 20, 2026 - 08:33:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[Apr 20, 2026 - 08:33:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 08:34:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121819 43.230663 42.992306 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 71.325195 72.925781 69.449219 +util 99.000000 100.000000 97.000000 +[Apr 20, 2026 - 08:34:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 351 : avg iteration time 60.53 seconds +[Apr 20, 2026 - 08:34:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 351 : data loading time 0.00 +[Apr 20, 2026 - 08:34:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 351 : avg forward pass time 17.54 +[Apr 20, 2026 - 08:34:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 351 : backward pass time 12.73 +[Apr 20, 2026 - 08:34:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 351 : optimizer step time 1.05 +[Apr 20, 2026 - 08:34:49 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 351-------------------- +[Apr 20, 2026 - 08:34:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0123 iter count: 1.0 +[Apr 20, 2026 - 08:34:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0123 iter count: 1.0 +[Apr 20, 2026 - 08:34:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 08:34:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 08:35:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121819 43.230663 42.992306 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 71.173828 72.847656 69.468750 +util 95.000000 100.000000 91.000000 +[Apr 20, 2026 - 08:35:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 352 : avg iteration time 63.49 seconds +[Apr 20, 2026 - 08:35:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 352 : data loading time 0.00 +[Apr 20, 2026 - 08:35:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 352 : avg forward pass time 19.49 +[Apr 20, 2026 - 08:35:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 352 : backward pass time 11.58 +[Apr 20, 2026 - 08:35:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 352 : optimizer step time 1.02 +[Apr 20, 2026 - 08:35:53 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 352-------------------- +[Apr 20, 2026 - 08:35:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0126 iter count: 1.0 +[Apr 20, 2026 - 08:35:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0126 iter count: 1.0 +[Apr 20, 2026 - 08:35:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 08:35:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 08:36:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121821 43.230667 42.992310 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 71.159180 72.847656 69.410156 +util 97.000000 100.000000 92.000000 +[Apr 20, 2026 - 08:36:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 353 : avg iteration time 55.65 seconds +[Apr 20, 2026 - 08:36:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 353 : data loading time 0.00 +[Apr 20, 2026 - 08:36:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 353 : avg forward pass time 15.75 +[Apr 20, 2026 - 08:36:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 353 : backward pass time 11.54 +[Apr 20, 2026 - 08:36:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 353 : optimizer step time 1.01 +[Apr 20, 2026 - 08:36:48 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 353-------------------- +[Apr 20, 2026 - 08:36:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0133 iter count: 1.0 +[Apr 20, 2026 - 08:36:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0133 iter count: 1.0 +[Apr 20, 2026 - 08:36:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 08:36:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 08:37:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121834 43.230667 42.992325 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 71.247070 72.847656 69.449219 +util 97.500000 99.000000 95.000000 +[Apr 20, 2026 - 08:37:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 354 : avg iteration time 50.17 seconds +[Apr 20, 2026 - 08:37:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 354 : data loading time 0.00 +[Apr 20, 2026 - 08:37:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 354 : avg forward pass time 12.47 +[Apr 20, 2026 - 08:37:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 354 : backward pass time 11.58 +[Apr 20, 2026 - 08:37:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 354 : optimizer step time 0.96 +[Apr 20, 2026 - 08:37:38 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 354-------------------- +[Apr 20, 2026 - 08:37:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0104 iter count: 1.0 +[Apr 20, 2026 - 08:37:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[Apr 20, 2026 - 08:37:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 08:37:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 08:40:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121638 43.229939 42.992306 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.506836 135.554688 135.439453 +util 69.000000 75.000000 65.000000 +[Apr 20, 2026 - 08:40:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 355 : avg iteration time 189.97 seconds +[Apr 20, 2026 - 08:40:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 355 : data loading time 0.00 +[Apr 20, 2026 - 08:40:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 355 : avg forward pass time 86.46 +[Apr 20, 2026 - 08:40:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 355 : backward pass time 8.40 +[Apr 20, 2026 - 08:40:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 355 : optimizer step time 0.34 +[Apr 20, 2026 - 08:40:48 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 355-------------------- +[Apr 20, 2026 - 08:40:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 118278.0443 iter count: 1.0 +[Apr 20, 2026 - 08:40:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6783 iter count: 1.0 +[Apr 20, 2026 - 08:40:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 118278.0443iter count: 1.0 +[Apr 20, 2026 - 08:40:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 08:40:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.3513iter count: 1.0 +[Apr 20, 2026 - 08:40:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.3793 iter count: 1.0 +[Apr 20, 2026 - 08:40:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.0749 iter count: 1.0 +[Apr 20, 2026 - 08:40:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 150701.6875iter count: 1.0 +[Apr 20, 2026 - 08:40:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 15.7767iter count: 1.0 +[Apr 20, 2026 - 08:40:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 1396967.8750iter count: 1.0 +[Apr 20, 2026 - 08:40:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6783iter count: 1.0 +[Apr 20, 2026 - 08:40:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.9120 iter count: 1.0 +[Apr 20, 2026 - 08:40:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 18473.1895iter count: 1.0 +[Apr 20, 2026 - 08:40:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.8966 iter count: 1.0 +[Apr 20, 2026 - 08:40:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 2423.9917iter count: 1.0 +[Apr 20, 2026 - 08:40:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 3.6244 iter count: 1.0 +[Apr 20, 2026 - 08:40:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 1406.4675iter count: 1.0 +[Apr 20, 2026 - 08:40:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.9152 iter count: 1.0 +[Apr 20, 2026 - 08:40:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 18589.8652iter count: 1.0 +[Apr 20, 2026 - 08:40:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[Apr 20, 2026 - 08:40:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 08:41:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121638 43.229939 42.992306 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.845215 69.976562 69.568359 +util 97.250000 99.000000 93.000000 +[Apr 20, 2026 - 08:41:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 356 : avg iteration time 60.46 seconds +[Apr 20, 2026 - 08:41:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 356 : data loading time 0.00 +[Apr 20, 2026 - 08:41:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 356 : avg forward pass time 17.51 +[Apr 20, 2026 - 08:41:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 356 : backward pass time 12.76 +[Apr 20, 2026 - 08:41:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 356 : optimizer step time 1.07 +[Apr 20, 2026 - 08:41:49 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 356-------------------- +[Apr 20, 2026 - 08:41:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0130 iter count: 1.0 +[Apr 20, 2026 - 08:41:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0130 iter count: 1.0 +[Apr 20, 2026 - 08:41:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 08:41:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 08:42:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121939 43.231125 42.992306 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.776855 69.878906 69.509766 +util 97.500000 100.000000 92.000000 +[Apr 20, 2026 - 08:42:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 357 : avg iteration time 54.69 seconds +[Apr 20, 2026 - 08:42:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 357 : data loading time 0.00 +[Apr 20, 2026 - 08:42:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 357 : avg forward pass time 15.31 +[Apr 20, 2026 - 08:42:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 357 : backward pass time 11.53 +[Apr 20, 2026 - 08:42:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 357 : optimizer step time 0.99 +[Apr 20, 2026 - 08:42:43 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 357-------------------- +[Apr 20, 2026 - 08:42:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0090 iter count: 1.0 +[Apr 20, 2026 - 08:42:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[Apr 20, 2026 - 08:42:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 08:42:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 08:43:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122240 43.232311 42.992321 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.825684 69.976562 69.490234 +util 95.750000 99.000000 93.000000 +[Apr 20, 2026 - 08:43:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 358 : avg iteration time 56.45 seconds +[Apr 20, 2026 - 08:43:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 358 : data loading time 0.00 +[Apr 20, 2026 - 08:43:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 358 : avg forward pass time 15.53 +[Apr 20, 2026 - 08:43:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 358 : backward pass time 12.76 +[Apr 20, 2026 - 08:43:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 358 : optimizer step time 1.07 +[Apr 20, 2026 - 08:43:40 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 358-------------------- +[Apr 20, 2026 - 08:43:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0099 iter count: 1.0 +[Apr 20, 2026 - 08:43:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[Apr 20, 2026 - 08:43:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 08:43:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 08:44:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122287 43.232517 42.992306 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 71.300293 72.867188 69.509766 +util 96.000000 99.000000 91.000000 +[Apr 20, 2026 - 08:44:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 359 : avg iteration time 52.43 seconds +[Apr 20, 2026 - 08:44:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 359 : data loading time 0.00 +[Apr 20, 2026 - 08:44:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 359 : avg forward pass time 14.13 +[Apr 20, 2026 - 08:44:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 359 : backward pass time 11.59 +[Apr 20, 2026 - 08:44:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 359 : optimizer step time 0.99 +[Apr 20, 2026 - 08:44:32 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 359-------------------- +[Apr 20, 2026 - 08:44:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0092 iter count: 1.0 +[Apr 20, 2026 - 08:44:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[Apr 20, 2026 - 08:44:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 08:44:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 08:47:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121706 43.230209 42.992306 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.467773 135.498047 135.457031 +util 70.250000 76.000000 61.000000 +[Apr 20, 2026 - 08:47:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 360 : avg iteration time 184.29 seconds +[Apr 20, 2026 - 08:47:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 360 : data loading time 0.00 +[Apr 20, 2026 - 08:47:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 360 : avg forward pass time 83.64 +[Apr 20, 2026 - 08:47:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 360 : backward pass time 8.39 +[Apr 20, 2026 - 08:47:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 360 : optimizer step time 0.34 +[Apr 20, 2026 - 08:47:37 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 360-------------------- +[Apr 20, 2026 - 08:47:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 107214.7676 iter count: 1.0 +[Apr 20, 2026 - 08:47:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6281 iter count: 1.0 +[Apr 20, 2026 - 08:47:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 107214.7676iter count: 1.0 +[Apr 20, 2026 - 08:47:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 08:47:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.1893iter count: 1.0 +[Apr 20, 2026 - 08:47:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.4827 iter count: 1.0 +[Apr 20, 2026 - 08:47:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.8977 iter count: 1.0 +[Apr 20, 2026 - 08:47:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 345628.0312iter count: 1.0 +[Apr 20, 2026 - 08:47:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 2.6259 iter count: 1.0 +[Apr 20, 2026 - 08:47:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 7241816.0000iter count: 1.0 +[Apr 20, 2026 - 08:47:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6281iter count: 1.0 +[Apr 20, 2026 - 08:47:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 7.8977 iter count: 1.0 +[Apr 20, 2026 - 08:47:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 7241816.0000iter count: 1.0 +[Apr 20, 2026 - 08:47:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.4598 iter count: 1.0 +[Apr 20, 2026 - 08:47:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 1012.0092iter count: 1.0 +[Apr 20, 2026 - 08:47:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 7.1602 iter count: 1.0 +[Apr 20, 2026 - 08:47:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 1656768.0000iter count: 1.0 +[Apr 20, 2026 - 08:47:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 5.8197 iter count: 1.0 +[Apr 20, 2026 - 08:47:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 113477.5625iter count: 1.0 +[Apr 20, 2026 - 08:47:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[Apr 20, 2026 - 08:47:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 08:48:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121760 43.230427 42.992306 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.752441 69.917969 69.373047 +util 97.750000 100.000000 93.000000 +[Apr 20, 2026 - 08:48:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 361 : avg iteration time 56.22 seconds +[Apr 20, 2026 - 08:48:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 361 : data loading time 0.00 +[Apr 20, 2026 - 08:48:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 361 : avg forward pass time 16.04 +[Apr 20, 2026 - 08:48:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 361 : backward pass time 11.56 +[Apr 20, 2026 - 08:48:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 361 : optimizer step time 0.98 +[Apr 20, 2026 - 08:48:33 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 361-------------------- +[Apr 20, 2026 - 08:48:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0121 iter count: 1.0 +[Apr 20, 2026 - 08:48:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[Apr 20, 2026 - 08:48:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 08:48:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 08:49:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121994 43.231361 42.992306 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.933105 70.017578 69.878906 +util 96.750000 100.000000 92.000000 +[Apr 20, 2026 - 08:49:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 362 : avg iteration time 53.70 seconds +[Apr 20, 2026 - 08:49:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 362 : data loading time 0.00 +[Apr 20, 2026 - 08:49:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 362 : avg forward pass time 14.18 +[Apr 20, 2026 - 08:49:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 362 : backward pass time 12.75 +[Apr 20, 2026 - 08:49:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 362 : optimizer step time 1.04 +[Apr 20, 2026 - 08:49:27 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 362-------------------- +[Apr 20, 2026 - 08:49:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0080 iter count: 1.0 +[Apr 20, 2026 - 08:49:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0080 iter count: 1.0 +[Apr 20, 2026 - 08:49:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 08:49:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 08:50:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122055 43.231606 42.992306 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.776855 69.859375 69.529297 +util 97.500000 100.000000 93.000000 +[Apr 20, 2026 - 08:50:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 363 : avg iteration time 47.01 seconds +[Apr 20, 2026 - 08:50:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 363 : data loading time 0.00 +[Apr 20, 2026 - 08:50:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 363 : avg forward pass time 11.47 +[Apr 20, 2026 - 08:50:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 363 : backward pass time 11.53 +[Apr 20, 2026 - 08:50:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 363 : optimizer step time 1.00 +[Apr 20, 2026 - 08:50:14 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 363-------------------- +[Apr 20, 2026 - 08:50:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0086 iter count: 1.0 +[Apr 20, 2026 - 08:50:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[Apr 20, 2026 - 08:50:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 08:50:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 08:51:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122057 43.231586 42.992310 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.869629 70.035156 69.548828 +util 97.750000 100.000000 94.000000 +[Apr 20, 2026 - 08:51:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 364 : avg iteration time 55.41 seconds +[Apr 20, 2026 - 08:51:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 364 : data loading time 0.00 +[Apr 20, 2026 - 08:51:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 364 : avg forward pass time 15.16 +[Apr 20, 2026 - 08:51:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 364 : backward pass time 11.54 +[Apr 20, 2026 - 08:51:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 364 : optimizer step time 0.99 +[Apr 20, 2026 - 08:51:09 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 364-------------------- +[Apr 20, 2026 - 08:51:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0084 iter count: 1.0 +[Apr 20, 2026 - 08:51:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[Apr 20, 2026 - 08:51:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 08:51:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 08:54:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121738 43.230312 42.992310 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.483887 135.517578 135.458984 +util 67.500000 71.000000 63.000000 +[Apr 20, 2026 - 08:54:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 365 : avg iteration time 187.78 seconds +[Apr 20, 2026 - 08:54:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 365 : data loading time 0.00 +[Apr 20, 2026 - 08:54:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 365 : avg forward pass time 85.37 +[Apr 20, 2026 - 08:54:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 365 : backward pass time 8.37 +[Apr 20, 2026 - 08:54:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 365 : optimizer step time 0.33 +[Apr 20, 2026 - 08:54:17 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 365-------------------- +[Apr 20, 2026 - 08:54:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 528400.8047 iter count: 1.0 +[Apr 20, 2026 - 08:54:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.7501 iter count: 1.0 +[Apr 20, 2026 - 08:54:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 528400.8047iter count: 1.0 +[Apr 20, 2026 - 08:54:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 08:54:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.8889iter count: 1.0 +[Apr 20, 2026 - 08:54:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.7365 iter count: 1.0 +[Apr 20, 2026 - 08:54:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 8.5645 iter count: 1.0 +[Apr 20, 2026 - 08:54:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 1162104.6250iter count: 1.0 +[Apr 20, 2026 - 08:54:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 32.2364iter count: 1.0 +[Apr 20, 2026 - 08:54:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 27480320.0000iter count: 1.0 +[Apr 20, 2026 - 08:54:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.7501iter count: 1.0 +[Apr 20, 2026 - 08:54:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.6411 iter count: 1.0 +[Apr 20, 2026 - 08:54:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 10745.8223iter count: 1.0 +[Apr 20, 2026 - 08:54:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 6.1250 iter count: 1.0 +[Apr 20, 2026 - 08:54:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 208986.8750iter count: 1.0 +[Apr 20, 2026 - 08:54:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.1465 iter count: 1.0 +[Apr 20, 2026 - 08:54:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 3995.6104iter count: 1.0 +[Apr 20, 2026 - 08:54:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.7808 iter count: 1.0 +[Apr 20, 2026 - 08:54:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 14209.0459iter count: 1.0 +[Apr 20, 2026 - 08:54:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[Apr 20, 2026 - 08:54:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 08:55:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121800 43.230564 42.992329 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.840332 70.037109 69.605469 +util 95.750000 99.000000 93.000000 +[Apr 20, 2026 - 08:55:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 366 : avg iteration time 53.36 seconds +[Apr 20, 2026 - 08:55:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 366 : data loading time 0.00 +[Apr 20, 2026 - 08:55:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 366 : avg forward pass time 14.61 +[Apr 20, 2026 - 08:55:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 366 : backward pass time 11.54 +[Apr 20, 2026 - 08:55:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 366 : optimizer step time 0.98 +[Apr 20, 2026 - 08:55:10 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 366-------------------- +[Apr 20, 2026 - 08:55:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0091 iter count: 1.0 +[Apr 20, 2026 - 08:55:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[Apr 20, 2026 - 08:55:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 08:55:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 08:56:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121858 43.230812 42.992310 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.889160 70.015625 69.742188 +util 96.250000 100.000000 91.000000 +[Apr 20, 2026 - 08:56:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 367 : avg iteration time 54.18 seconds +[Apr 20, 2026 - 08:56:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 367 : data loading time 0.00 +[Apr 20, 2026 - 08:56:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 367 : avg forward pass time 14.26 +[Apr 20, 2026 - 08:56:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 367 : backward pass time 12.83 +[Apr 20, 2026 - 08:56:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 367 : optimizer step time 1.27 +[Apr 20, 2026 - 08:56:04 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 367-------------------- +[Apr 20, 2026 - 08:56:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0089 iter count: 1.0 +[Apr 20, 2026 - 08:56:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[Apr 20, 2026 - 08:56:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 08:56:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 08:57:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121860 43.230820 42.992310 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.781738 69.859375 69.548828 +util 96.500000 100.000000 92.000000 +[Apr 20, 2026 - 08:57:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 368 : avg iteration time 58.06 seconds +[Apr 20, 2026 - 08:57:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 368 : data loading time 0.00 +[Apr 20, 2026 - 08:57:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 368 : avg forward pass time 16.98 +[Apr 20, 2026 - 08:57:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 368 : backward pass time 11.56 +[Apr 20, 2026 - 08:57:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 368 : optimizer step time 0.99 +[Apr 20, 2026 - 08:57:02 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 368-------------------- +[Apr 20, 2026 - 08:57:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0107 iter count: 1.0 +[Apr 20, 2026 - 08:57:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[Apr 20, 2026 - 08:57:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 08:57:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 08:58:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121863 43.230812 42.992310 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.801270 69.937500 69.529297 +util 95.750000 99.000000 91.000000 +[Apr 20, 2026 - 08:58:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 369 : avg iteration time 62.13 seconds +[Apr 20, 2026 - 08:58:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 369 : data loading time 0.00 +[Apr 20, 2026 - 08:58:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 369 : avg forward pass time 18.35 +[Apr 20, 2026 - 08:58:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 369 : backward pass time 12.77 +[Apr 20, 2026 - 08:58:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 369 : optimizer step time 1.07 +[Apr 20, 2026 - 08:58:04 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 369-------------------- +[Apr 20, 2026 - 08:58:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0095 iter count: 1.0 +[Apr 20, 2026 - 08:58:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[Apr 20, 2026 - 08:58:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 08:58:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 09:01:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121462 43.229221 42.992310 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.497559 135.574219 135.419922 +util 73.000000 76.000000 69.000000 +[Apr 20, 2026 - 09:01:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 370 : avg iteration time 175.93 seconds +[Apr 20, 2026 - 09:01:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 370 : data loading time 0.00 +[Apr 20, 2026 - 09:01:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 370 : avg forward pass time 79.44 +[Apr 20, 2026 - 09:01:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 370 : backward pass time 8.42 +[Apr 20, 2026 - 09:01:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 370 : optimizer step time 0.33 +[Apr 20, 2026 - 09:01:00 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 370-------------------- +[Apr 20, 2026 - 09:01:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 53436.6851 iter count: 1.0 +[Apr 20, 2026 - 09:01:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6160 iter count: 1.0 +[Apr 20, 2026 - 09:01:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 53436.6851iter count: 1.0 +[Apr 20, 2026 - 09:01:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 09:01:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.3198iter count: 1.0 +[Apr 20, 2026 - 09:01:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.8085 iter count: 1.0 +[Apr 20, 2026 - 09:01:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.1152 iter count: 1.0 +[Apr 20, 2026 - 09:01:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 109922.7500iter count: 1.0 +[Apr 20, 2026 - 09:01:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 5.0384 iter count: 1.0 +[Apr 20, 2026 - 09:01:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 1514161.6250iter count: 1.0 +[Apr 20, 2026 - 09:01:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6160iter count: 1.0 +[Apr 20, 2026 - 09:01:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.4956 iter count: 1.0 +[Apr 20, 2026 - 09:01:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 1087.1283iter count: 1.0 +[Apr 20, 2026 - 09:01:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 5.9753 iter count: 1.0 +[Apr 20, 2026 - 09:01:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 154910.0156iter count: 1.0 +[Apr 20, 2026 - 09:01:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 2.2191 iter count: 1.0 +[Apr 20, 2026 - 09:01:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 84.6260 iter count: 1.0 +[Apr 20, 2026 - 09:01:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 5.6452 iter count: 1.0 +[Apr 20, 2026 - 09:01:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 80042.7109iter count: 1.0 +[Apr 20, 2026 - 09:01:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0080 iter count: 1.0 +[Apr 20, 2026 - 09:01:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 09:01:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121468 43.229221 42.992310 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.776855 69.917969 69.410156 +util 98.000000 99.000000 95.000000 +[Apr 20, 2026 - 09:01:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 371 : avg iteration time 54.38 seconds +[Apr 20, 2026 - 09:01:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 371 : data loading time 0.00 +[Apr 20, 2026 - 09:01:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 371 : avg forward pass time 14.55 +[Apr 20, 2026 - 09:01:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 371 : backward pass time 11.53 +[Apr 20, 2026 - 09:01:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 371 : optimizer step time 1.08 +[Apr 20, 2026 - 09:01:55 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 371-------------------- +[Apr 20, 2026 - 09:01:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0082 iter count: 1.0 +[Apr 20, 2026 - 09:01:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[Apr 20, 2026 - 09:01:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 09:01:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 09:02:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121887 43.230923 42.992310 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.786621 69.839844 69.644531 +util 96.250000 99.000000 92.000000 +[Apr 20, 2026 - 09:02:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 372 : avg iteration time 58.15 seconds +[Apr 20, 2026 - 09:02:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 372 : data loading time 0.00 +[Apr 20, 2026 - 09:02:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 372 : avg forward pass time 17.03 +[Apr 20, 2026 - 09:02:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 372 : backward pass time 11.55 +[Apr 20, 2026 - 09:02:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 372 : optimizer step time 0.99 +[Apr 20, 2026 - 09:02:53 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 372-------------------- +[Apr 20, 2026 - 09:02:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0112 iter count: 1.0 +[Apr 20, 2026 - 09:02:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[Apr 20, 2026 - 09:02:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 09:02:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 09:03:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121887 43.230923 42.992310 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.850098 69.996094 69.585938 +util 99.500000 100.000000 98.000000 +[Apr 20, 2026 - 09:03:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 373 : avg iteration time 47.88 seconds +[Apr 20, 2026 - 09:03:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 373 : data loading time 0.00 +[Apr 20, 2026 - 09:03:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 373 : avg forward pass time 11.27 +[Apr 20, 2026 - 09:03:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 373 : backward pass time 11.54 +[Apr 20, 2026 - 09:03:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 373 : optimizer step time 0.99 +[Apr 20, 2026 - 09:03:41 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 373-------------------- +[Apr 20, 2026 - 09:03:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0080 iter count: 1.0 +[Apr 20, 2026 - 09:03:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0080 iter count: 1.0 +[Apr 20, 2026 - 09:03:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 09:03:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 09:04:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121887 43.230923 42.992310 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.864746 70.054688 69.488281 +util 97.000000 100.000000 92.000000 +[Apr 20, 2026 - 09:04:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 374 : avg iteration time 56.66 seconds +[Apr 20, 2026 - 09:04:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 374 : data loading time 0.00 +[Apr 20, 2026 - 09:04:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 374 : avg forward pass time 15.23 +[Apr 20, 2026 - 09:04:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 374 : backward pass time 12.11 +[Apr 20, 2026 - 09:04:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 374 : optimizer step time 1.55 +[Apr 20, 2026 - 09:04:37 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 374-------------------- +[Apr 20, 2026 - 09:04:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0111 iter count: 1.0 +[Apr 20, 2026 - 09:04:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[Apr 20, 2026 - 09:04:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 09:04:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 09:07:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121714 43.229973 42.992359 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.532715 135.556641 135.478516 +util 70.500000 76.000000 61.000000 +[Apr 20, 2026 - 09:07:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 375 : avg iteration time 176.34 seconds +[Apr 20, 2026 - 09:07:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 375 : data loading time 0.00 +[Apr 20, 2026 - 09:07:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 375 : avg forward pass time 79.04 +[Apr 20, 2026 - 09:07:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 375 : backward pass time 8.41 +[Apr 20, 2026 - 09:07:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 375 : optimizer step time 0.33 +[Apr 20, 2026 - 09:07:34 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 375-------------------- +[Apr 20, 2026 - 09:07:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 2246018.1152 iter count: 1.0 +[Apr 20, 2026 - 09:07:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6376 iter count: 1.0 +[Apr 20, 2026 - 09:07:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 2246018.1152iter count: 1.0 +[Apr 20, 2026 - 09:07:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 09:07:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.9753iter count: 1.0 +[Apr 20, 2026 - 09:07:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.1039 iter count: 1.0 +[Apr 20, 2026 - 09:07:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 8.8625 iter count: 1.0 +[Apr 20, 2026 - 09:07:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 3040511.0000iter count: 1.0 +[Apr 20, 2026 - 09:07:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 9.0949 iter count: 1.0 +[Apr 20, 2026 - 09:07:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 49877972.0000iter count: 1.0 +[Apr 20, 2026 - 09:07:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6376iter count: 1.0 +[Apr 20, 2026 - 09:07:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.4724 iter count: 1.0 +[Apr 20, 2026 - 09:07:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 1037.6636iter count: 1.0 +[Apr 20, 2026 - 09:07:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.6664 iter count: 1.0 +[Apr 20, 2026 - 09:07:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 11302.8848iter count: 1.0 +[Apr 20, 2026 - 09:07:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 3.2036 iter count: 1.0 +[Apr 20, 2026 - 09:07:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 606.1973iter count: 1.0 +[Apr 20, 2026 - 09:07:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.7317 iter count: 1.0 +[Apr 20, 2026 - 09:07:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 12879.0342iter count: 1.0 +[Apr 20, 2026 - 09:07:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[Apr 20, 2026 - 09:07:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 09:08:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121778 43.230213 42.992359 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.465332 72.554688 69.470703 +util 98.750000 100.000000 96.000000 +[Apr 20, 2026 - 09:08:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 376 : avg iteration time 60.36 seconds +[Apr 20, 2026 - 09:08:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 376 : data loading time 0.00 +[Apr 20, 2026 - 09:08:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 376 : avg forward pass time 17.21 +[Apr 20, 2026 - 09:08:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 376 : backward pass time 11.62 +[Apr 20, 2026 - 09:08:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 376 : optimizer step time 1.39 +[Apr 20, 2026 - 09:08:34 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 376-------------------- +[Apr 20, 2026 - 09:08:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0104 iter count: 1.0 +[Apr 20, 2026 - 09:08:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[Apr 20, 2026 - 09:08:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 09:08:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 09:09:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122017 43.231201 42.992344 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.577637 72.593750 69.802734 +util 95.750000 100.000000 90.000000 +[Apr 20, 2026 - 09:09:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 377 : avg iteration time 52.68 seconds +[Apr 20, 2026 - 09:09:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 377 : data loading time 0.00 +[Apr 20, 2026 - 09:09:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 377 : avg forward pass time 14.27 +[Apr 20, 2026 - 09:09:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 377 : backward pass time 11.59 +[Apr 20, 2026 - 09:09:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 377 : optimizer step time 0.97 +[Apr 20, 2026 - 09:09:27 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 377-------------------- +[Apr 20, 2026 - 09:09:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0097 iter count: 1.0 +[Apr 20, 2026 - 09:09:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[Apr 20, 2026 - 09:09:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 09:09:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 09:10:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122141 43.231697 42.992344 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.558105 72.515625 69.802734 +util 99.750000 100.000000 99.000000 +[Apr 20, 2026 - 09:10:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 378 : avg iteration time 52.36 seconds +[Apr 20, 2026 - 09:10:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 378 : data loading time 0.00 +[Apr 20, 2026 - 09:10:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 378 : avg forward pass time 14.01 +[Apr 20, 2026 - 09:10:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 378 : backward pass time 11.57 +[Apr 20, 2026 - 09:10:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 378 : optimizer step time 0.99 +[Apr 20, 2026 - 09:10:19 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 378-------------------- +[Apr 20, 2026 - 09:10:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0089 iter count: 1.0 +[Apr 20, 2026 - 09:10:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[Apr 20, 2026 - 09:10:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 09:10:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 09:11:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122141 43.231697 42.992344 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.840332 69.958984 69.488281 +util 98.750000 100.000000 97.000000 +[Apr 20, 2026 - 09:11:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 379 : avg iteration time 56.79 seconds +[Apr 20, 2026 - 09:11:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 379 : data loading time 0.00 +[Apr 20, 2026 - 09:11:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 379 : avg forward pass time 15.60 +[Apr 20, 2026 - 09:11:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 379 : backward pass time 12.87 +[Apr 20, 2026 - 09:11:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 379 : optimizer step time 1.10 +[Apr 20, 2026 - 09:11:16 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 379-------------------- +[Apr 20, 2026 - 09:11:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0077 iter count: 1.0 +[Apr 20, 2026 - 09:11:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0077 iter count: 1.0 +[Apr 20, 2026 - 09:11:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 09:11:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 09:14:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122028 43.230793 42.992535 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.512207 135.576172 135.439453 +util 72.250000 78.000000 63.000000 +[Apr 20, 2026 - 09:14:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 380 : avg iteration time 193.27 seconds +[Apr 20, 2026 - 09:14:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 380 : data loading time 0.00 +[Apr 20, 2026 - 09:14:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 380 : avg forward pass time 87.43 +[Apr 20, 2026 - 09:14:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 380 : backward pass time 9.76 +[Apr 20, 2026 - 09:14:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 380 : optimizer step time 0.35 +[Apr 20, 2026 - 09:14:29 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 380-------------------- +[Apr 20, 2026 - 09:14:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 115698.8130 iter count: 1.0 +[Apr 20, 2026 - 09:14:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6153 iter count: 1.0 +[Apr 20, 2026 - 09:14:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 115698.8130iter count: 1.0 +[Apr 20, 2026 - 09:14:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 09:14:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.4432iter count: 1.0 +[Apr 20, 2026 - 09:14:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.8034 iter count: 1.0 +[Apr 20, 2026 - 09:14:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.4788 iter count: 1.0 +[Apr 20, 2026 - 09:14:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 214213.1562iter count: 1.0 +[Apr 20, 2026 - 09:14:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 36.8462iter count: 1.0 +[Apr 20, 2026 - 09:14:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 3133286.5000iter count: 1.0 +[Apr 20, 2026 - 09:14:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6153iter count: 1.0 +[Apr 20, 2026 - 09:14:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.2343 iter count: 1.0 +[Apr 20, 2026 - 09:14:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 644.5544iter count: 1.0 +[Apr 20, 2026 - 09:14:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 5.1131 iter count: 1.0 +[Apr 20, 2026 - 09:14:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 27619.4219iter count: 1.0 +[Apr 20, 2026 - 09:14:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 3.1544 iter count: 1.0 +[Apr 20, 2026 - 09:14:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 549.4291iter count: 1.0 +[Apr 20, 2026 - 09:14:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 3.6648 iter count: 1.0 +[Apr 20, 2026 - 09:14:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 1524.9132iter count: 1.0 +[Apr 20, 2026 - 09:14:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0080 iter count: 1.0 +[Apr 20, 2026 - 09:14:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 09:15:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122022 43.230793 42.992535 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.703613 69.878906 69.373047 +util 96.000000 100.000000 92.000000 +[Apr 20, 2026 - 09:15:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 381 : avg iteration time 53.67 seconds +[Apr 20, 2026 - 09:15:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 381 : data loading time 0.00 +[Apr 20, 2026 - 09:15:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 381 : avg forward pass time 14.79 +[Apr 20, 2026 - 09:15:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 381 : backward pass time 11.56 +[Apr 20, 2026 - 09:15:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 381 : optimizer step time 0.98 +[Apr 20, 2026 - 09:15:23 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 381-------------------- +[Apr 20, 2026 - 09:15:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0085 iter count: 1.0 +[Apr 20, 2026 - 09:15:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[Apr 20, 2026 - 09:15:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 09:15:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 09:16:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122028 43.230793 42.992535 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.776855 69.917969 69.490234 +util 97.500000 100.000000 92.000000 +[Apr 20, 2026 - 09:16:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 382 : avg iteration time 47.71 seconds +[Apr 20, 2026 - 09:16:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 382 : data loading time 0.00 +[Apr 20, 2026 - 09:16:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 382 : avg forward pass time 11.12 +[Apr 20, 2026 - 09:16:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 382 : backward pass time 11.54 +[Apr 20, 2026 - 09:16:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 382 : optimizer step time 0.99 +[Apr 20, 2026 - 09:16:11 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 382-------------------- +[Apr 20, 2026 - 09:16:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0087 iter count: 1.0 +[Apr 20, 2026 - 09:16:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[Apr 20, 2026 - 09:16:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 09:16:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 09:17:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122085 43.231041 42.992535 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.825684 69.957031 69.490234 +util 99.250000 100.000000 97.000000 +[Apr 20, 2026 - 09:17:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 383 : avg iteration time 56.97 seconds +[Apr 20, 2026 - 09:17:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 383 : data loading time 0.00 +[Apr 20, 2026 - 09:17:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 383 : avg forward pass time 15.36 +[Apr 20, 2026 - 09:17:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 383 : backward pass time 12.52 +[Apr 20, 2026 - 09:17:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 383 : optimizer step time 1.55 +[Apr 20, 2026 - 09:17:08 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 383-------------------- +[Apr 20, 2026 - 09:17:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0115 iter count: 1.0 +[Apr 20, 2026 - 09:17:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[Apr 20, 2026 - 09:17:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 09:17:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 09:18:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122082 43.231030 42.992535 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.762207 69.878906 69.490234 +util 99.000000 100.000000 97.000000 +[Apr 20, 2026 - 09:18:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 384 : avg iteration time 63.60 seconds +[Apr 20, 2026 - 09:18:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 384 : data loading time 0.00 +[Apr 20, 2026 - 09:18:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 384 : avg forward pass time 19.75 +[Apr 20, 2026 - 09:18:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 384 : backward pass time 11.56 +[Apr 20, 2026 - 09:18:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 384 : optimizer step time 0.97 +[Apr 20, 2026 - 09:18:11 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 384-------------------- +[Apr 20, 2026 - 09:18:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0101 iter count: 1.0 +[Apr 20, 2026 - 09:18:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[Apr 20, 2026 - 09:18:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 09:18:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 09:21:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121894 43.230255 42.992535 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.521973 135.554688 135.460938 +util 78.750000 95.000000 68.000000 +[Apr 20, 2026 - 09:21:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 385 : avg iteration time 178.14 seconds +[Apr 20, 2026 - 09:21:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 385 : data loading time 0.00 +[Apr 20, 2026 - 09:21:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 385 : avg forward pass time 80.11 +[Apr 20, 2026 - 09:21:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 385 : backward pass time 9.26 +[Apr 20, 2026 - 09:21:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 385 : optimizer step time 0.34 +[Apr 20, 2026 - 09:21:09 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 385-------------------- +[Apr 20, 2026 - 09:21:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 44968.9238 iter count: 1.0 +[Apr 20, 2026 - 09:21:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5759 iter count: 1.0 +[Apr 20, 2026 - 09:21:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 44968.9238iter count: 1.0 +[Apr 20, 2026 - 09:21:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 09:21:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.2022iter count: 1.0 +[Apr 20, 2026 - 09:21:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 2.0077 iter count: 1.0 +[Apr 20, 2026 - 09:21:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.9555 iter count: 1.0 +[Apr 20, 2026 - 09:21:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 76447.5938iter count: 1.0 +[Apr 20, 2026 - 09:21:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 55.4477iter count: 1.0 +[Apr 20, 2026 - 09:21:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 1100182.7500iter count: 1.0 +[Apr 20, 2026 - 09:21:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5759iter count: 1.0 +[Apr 20, 2026 - 09:21:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 2.2095 iter count: 1.0 +[Apr 20, 2026 - 09:21:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 83.0055 iter count: 1.0 +[Apr 20, 2026 - 09:21:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.0378 iter count: 1.0 +[Apr 20, 2026 - 09:21:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 435.1336iter count: 1.0 +[Apr 20, 2026 - 09:21:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 5.4169 iter count: 1.0 +[Apr 20, 2026 - 09:21:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 50701.5430iter count: 1.0 +[Apr 20, 2026 - 09:21:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.5387 iter count: 1.0 +[Apr 20, 2026 - 09:21:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 8754.5068iter count: 1.0 +[Apr 20, 2026 - 09:21:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[Apr 20, 2026 - 09:21:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 09:22:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121892 43.230259 42.992535 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.693848 69.781250 69.529297 +util 95.500000 99.000000 91.000000 +[Apr 20, 2026 - 09:22:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 386 : avg iteration time 56.61 seconds +[Apr 20, 2026 - 09:22:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 386 : data loading time 0.00 +[Apr 20, 2026 - 09:22:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 386 : avg forward pass time 16.25 +[Apr 20, 2026 - 09:22:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 386 : backward pass time 11.55 +[Apr 20, 2026 - 09:22:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 386 : optimizer step time 0.99 +[Apr 20, 2026 - 09:22:06 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 386-------------------- +[Apr 20, 2026 - 09:22:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0083 iter count: 1.0 +[Apr 20, 2026 - 09:22:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[Apr 20, 2026 - 09:22:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 09:22:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 09:22:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122187 43.231449 42.992535 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.854980 69.996094 69.568359 +util 97.250000 100.000000 91.000000 +[Apr 20, 2026 - 09:22:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 387 : avg iteration time 53.10 seconds +[Apr 20, 2026 - 09:22:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 387 : data loading time 0.00 +[Apr 20, 2026 - 09:22:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 387 : avg forward pass time 13.91 +[Apr 20, 2026 - 09:22:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 387 : backward pass time 11.56 +[Apr 20, 2026 - 09:22:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 387 : optimizer step time 1.07 +[Apr 20, 2026 - 09:22:59 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 387-------------------- +[Apr 20, 2026 - 09:22:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0088 iter count: 1.0 +[Apr 20, 2026 - 09:22:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[Apr 20, 2026 - 09:22:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 09:22:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 09:23:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122189 43.231449 42.992542 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.791504 69.937500 69.431641 +util 96.750000 99.000000 94.000000 +[Apr 20, 2026 - 09:23:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 388 : avg iteration time 52.89 seconds +[Apr 20, 2026 - 09:23:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 388 : data loading time 0.00 +[Apr 20, 2026 - 09:23:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 388 : avg forward pass time 14.05 +[Apr 20, 2026 - 09:23:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 388 : backward pass time 12.16 +[Apr 20, 2026 - 09:23:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 388 : optimizer step time 1.01 +[Apr 20, 2026 - 09:23:52 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 388-------------------- +[Apr 20, 2026 - 09:23:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0088 iter count: 1.0 +[Apr 20, 2026 - 09:23:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[Apr 20, 2026 - 09:23:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 09:23:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 09:24:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122243 43.231670 42.992538 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.835449 69.937500 69.587891 +util 95.500000 100.000000 91.000000 +[Apr 20, 2026 - 09:24:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 389 : avg iteration time 58.20 seconds +[Apr 20, 2026 - 09:24:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 389 : data loading time 0.00 +[Apr 20, 2026 - 09:24:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 389 : avg forward pass time 16.78 +[Apr 20, 2026 - 09:24:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 389 : backward pass time 11.52 +[Apr 20, 2026 - 09:24:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 389 : optimizer step time 0.97 +[Apr 20, 2026 - 09:24:50 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 389-------------------- +[Apr 20, 2026 - 09:24:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0084 iter count: 1.0 +[Apr 20, 2026 - 09:24:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[Apr 20, 2026 - 09:24:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 09:24:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 09:27:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122024 43.230774 42.992546 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.469238 135.498047 135.382812 +util 69.250000 77.000000 65.000000 +[Apr 20, 2026 - 09:27:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 390 : avg iteration time 184.79 seconds +[Apr 20, 2026 - 09:27:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 390 : data loading time 0.00 +[Apr 20, 2026 - 09:27:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 390 : avg forward pass time 83.87 +[Apr 20, 2026 - 09:27:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 390 : backward pass time 8.40 +[Apr 20, 2026 - 09:27:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 390 : optimizer step time 0.33 +[Apr 20, 2026 - 09:27:55 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 390-------------------- +[Apr 20, 2026 - 09:27:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 260301.5156 iter count: 1.0 +[Apr 20, 2026 - 09:27:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6771 iter count: 1.0 +[Apr 20, 2026 - 09:27:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 260301.5156iter count: 1.0 +[Apr 20, 2026 - 09:27:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 09:27:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.4048iter count: 1.0 +[Apr 20, 2026 - 09:27:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.5436 iter count: 1.0 +[Apr 20, 2026 - 09:27:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.7980 iter count: 1.0 +[Apr 20, 2026 - 09:27:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 382834.5312iter count: 1.0 +[Apr 20, 2026 - 09:27:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 2.9662 iter count: 1.0 +[Apr 20, 2026 - 09:27:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 5933317.5000iter count: 1.0 +[Apr 20, 2026 - 09:27:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6771iter count: 1.0 +[Apr 20, 2026 - 09:27:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.4847 iter count: 1.0 +[Apr 20, 2026 - 09:27:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 1063.6619iter count: 1.0 +[Apr 20, 2026 - 09:27:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.8221 iter count: 1.0 +[Apr 20, 2026 - 09:27:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 15433.4258iter count: 1.0 +[Apr 20, 2026 - 09:27:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 3.7823 iter count: 1.0 +[Apr 20, 2026 - 09:27:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 1928.5573iter count: 1.0 +[Apr 20, 2026 - 09:27:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 3.3867 iter count: 1.0 +[Apr 20, 2026 - 09:27:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 874.3222iter count: 1.0 +[Apr 20, 2026 - 09:27:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[Apr 20, 2026 - 09:27:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 09:28:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122022 43.230774 42.992535 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.879395 69.939453 69.859375 +util 95.000000 99.000000 92.000000 +[Apr 20, 2026 - 09:28:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 391 : avg iteration time 56.78 seconds +[Apr 20, 2026 - 09:28:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 391 : data loading time 0.00 +[Apr 20, 2026 - 09:28:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 391 : avg forward pass time 16.19 +[Apr 20, 2026 - 09:28:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 391 : backward pass time 11.56 +[Apr 20, 2026 - 09:28:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 391 : optimizer step time 0.99 +[Apr 20, 2026 - 09:28:52 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 391-------------------- +[Apr 20, 2026 - 09:28:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0088 iter count: 1.0 +[Apr 20, 2026 - 09:28:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[Apr 20, 2026 - 09:28:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 09:28:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 09:29:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122019 43.230774 42.992535 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.918457 69.939453 69.859375 +util 97.250000 99.000000 92.000000 +[Apr 20, 2026 - 09:29:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 392 : avg iteration time 59.14 seconds +[Apr 20, 2026 - 09:29:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 392 : data loading time 0.00 +[Apr 20, 2026 - 09:29:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 392 : avg forward pass time 17.11 +[Apr 20, 2026 - 09:29:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 392 : backward pass time 12.36 +[Apr 20, 2026 - 09:29:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 392 : optimizer step time 1.00 +[Apr 20, 2026 - 09:29:51 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 392-------------------- +[Apr 20, 2026 - 09:29:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0091 iter count: 1.0 +[Apr 20, 2026 - 09:29:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[Apr 20, 2026 - 09:29:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 09:29:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 09:30:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122022 43.230774 42.992535 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.894043 69.996094 69.783203 +util 94.500000 99.000000 92.000000 +[Apr 20, 2026 - 09:30:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 393 : avg iteration time 46.26 seconds +[Apr 20, 2026 - 09:30:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 393 : data loading time 0.00 +[Apr 20, 2026 - 09:30:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 393 : avg forward pass time 11.08 +[Apr 20, 2026 - 09:30:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 393 : backward pass time 11.56 +[Apr 20, 2026 - 09:30:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 393 : optimizer step time 0.98 +[Apr 20, 2026 - 09:30:37 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 393-------------------- +[Apr 20, 2026 - 09:30:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0076 iter count: 1.0 +[Apr 20, 2026 - 09:30:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0076 iter count: 1.0 +[Apr 20, 2026 - 09:30:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 09:30:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 09:31:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122023 43.230782 42.992535 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.791504 69.898438 69.509766 +util 96.750000 100.000000 92.000000 +[Apr 20, 2026 - 09:31:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 394 : avg iteration time 53.01 seconds +[Apr 20, 2026 - 09:31:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 394 : data loading time 0.00 +[Apr 20, 2026 - 09:31:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 394 : avg forward pass time 14.11 +[Apr 20, 2026 - 09:31:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 394 : backward pass time 11.60 +[Apr 20, 2026 - 09:31:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 394 : optimizer step time 1.38 +[Apr 20, 2026 - 09:31:30 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 394-------------------- +[Apr 20, 2026 - 09:31:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0087 iter count: 1.0 +[Apr 20, 2026 - 09:31:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0087 iter count: 1.0 +[Apr 20, 2026 - 09:31:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 09:31:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 09:34:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122053 43.230774 42.992611 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.478516 135.517578 135.419922 +util 65.500000 70.000000 61.000000 +[Apr 20, 2026 - 09:34:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 395 : avg iteration time 181.25 seconds +[Apr 20, 2026 - 09:34:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 395 : data loading time 0.00 +[Apr 20, 2026 - 09:34:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 395 : avg forward pass time 82.04 +[Apr 20, 2026 - 09:34:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 395 : backward pass time 8.41 +[Apr 20, 2026 - 09:34:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 395 : optimizer step time 0.34 +[Apr 20, 2026 - 09:34:31 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 395-------------------- +[Apr 20, 2026 - 09:34:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 308141.9238 iter count: 1.0 +[Apr 20, 2026 - 09:34:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6865 iter count: 1.0 +[Apr 20, 2026 - 09:34:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 308141.9238iter count: 1.0 +[Apr 20, 2026 - 09:34:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 09:34:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.6508iter count: 1.0 +[Apr 20, 2026 - 09:34:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 2.1853 iter count: 1.0 +[Apr 20, 2026 - 09:34:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.7797 iter count: 1.0 +[Apr 20, 2026 - 09:34:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 479156.5312iter count: 1.0 +[Apr 20, 2026 - 09:34:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 79.0986iter count: 1.0 +[Apr 20, 2026 - 09:34:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 5720046.0000iter count: 1.0 +[Apr 20, 2026 - 09:34:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6865iter count: 1.0 +[Apr 20, 2026 - 09:34:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.7577 iter count: 1.0 +[Apr 20, 2026 - 09:34:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 1836.0702iter count: 1.0 +[Apr 20, 2026 - 09:34:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.7337 iter count: 1.0 +[Apr 20, 2026 - 09:34:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 12930.0820iter count: 1.0 +[Apr 20, 2026 - 09:34:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.2022 iter count: 1.0 +[Apr 20, 2026 - 09:34:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 4466.8130iter count: 1.0 +[Apr 20, 2026 - 09:34:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 5.7078 iter count: 1.0 +[Apr 20, 2026 - 09:34:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 90728.1328iter count: 1.0 +[Apr 20, 2026 - 09:34:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[Apr 20, 2026 - 09:34:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 09:35:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122111 43.231022 42.992596 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 71.305176 72.867188 69.587891 +util 97.000000 99.000000 93.000000 +[Apr 20, 2026 - 09:35:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 396 : avg iteration time 57.81 seconds +[Apr 20, 2026 - 09:35:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 396 : data loading time 0.00 +[Apr 20, 2026 - 09:35:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 396 : avg forward pass time 16.25 +[Apr 20, 2026 - 09:35:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 396 : backward pass time 12.08 +[Apr 20, 2026 - 09:35:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 396 : optimizer step time 1.55 +[Apr 20, 2026 - 09:35:29 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 396-------------------- +[Apr 20, 2026 - 09:35:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0085 iter count: 1.0 +[Apr 20, 2026 - 09:35:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0085 iter count: 1.0 +[Apr 20, 2026 - 09:35:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 09:35:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 09:36:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122120 43.231022 42.992615 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.776855 69.878906 69.587891 +util 97.750000 100.000000 92.000000 +[Apr 20, 2026 - 09:36:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 397 : avg iteration time 53.10 seconds +[Apr 20, 2026 - 09:36:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 397 : data loading time 0.00 +[Apr 20, 2026 - 09:36:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 397 : avg forward pass time 14.22 +[Apr 20, 2026 - 09:36:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 397 : backward pass time 11.86 +[Apr 20, 2026 - 09:36:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 397 : optimizer step time 0.97 +[Apr 20, 2026 - 09:36:22 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 397-------------------- +[Apr 20, 2026 - 09:36:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0088 iter count: 1.0 +[Apr 20, 2026 - 09:36:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[Apr 20, 2026 - 09:36:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 09:36:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 09:37:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122111 43.231022 42.992596 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.845215 69.880859 69.820312 +util 94.750000 99.000000 91.000000 +[Apr 20, 2026 - 09:37:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 398 : avg iteration time 52.84 seconds +[Apr 20, 2026 - 09:37:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 398 : data loading time 0.00 +[Apr 20, 2026 - 09:37:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 398 : avg forward pass time 14.00 +[Apr 20, 2026 - 09:37:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 398 : backward pass time 11.55 +[Apr 20, 2026 - 09:37:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 398 : optimizer step time 0.97 +[Apr 20, 2026 - 09:37:15 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 398-------------------- +[Apr 20, 2026 - 09:37:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0090 iter count: 1.0 +[Apr 20, 2026 - 09:37:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[Apr 20, 2026 - 09:37:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 09:37:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 09:38:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122172 43.231266 42.992596 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.908691 70.056641 69.781250 +util 94.500000 99.000000 92.000000 +[Apr 20, 2026 - 09:38:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 399 : avg iteration time 62.41 seconds +[Apr 20, 2026 - 09:38:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 399 : data loading time 0.00 +[Apr 20, 2026 - 09:38:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 399 : avg forward pass time 18.36 +[Apr 20, 2026 - 09:38:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 399 : backward pass time 13.00 +[Apr 20, 2026 - 09:38:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 399 : optimizer step time 1.17 +[Apr 20, 2026 - 09:38:18 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 399-------------------- +[Apr 20, 2026 - 09:38:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0096 iter count: 1.0 +[Apr 20, 2026 - 09:38:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[Apr 20, 2026 - 09:38:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 09:38:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 09:41:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121786 43.229710 42.992596 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.531738 135.539062 135.517578 +util 69.750000 78.000000 66.000000 +[Apr 20, 2026 - 09:41:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 400 : avg iteration time 187.44 seconds +[Apr 20, 2026 - 09:41:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 400 : data loading time 0.00 +[Apr 20, 2026 - 09:41:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 400 : avg forward pass time 85.22 +[Apr 20, 2026 - 09:41:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 400 : backward pass time 8.39 +[Apr 20, 2026 - 09:41:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 400 : optimizer step time 0.33 +[Apr 20, 2026 - 09:41:25 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 400-------------------- +[Apr 20, 2026 - 09:41:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 158628.5439 iter count: 1.0 +[Apr 20, 2026 - 09:41:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6261 iter count: 1.0 +[Apr 20, 2026 - 09:41:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 158628.5439iter count: 1.0 +[Apr 20, 2026 - 09:41:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 09:41:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.7432iter count: 1.0 +[Apr 20, 2026 - 09:41:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.5048 iter count: 1.0 +[Apr 20, 2026 - 09:41:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.4318 iter count: 1.0 +[Apr 20, 2026 - 09:41:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 263893.0312iter count: 1.0 +[Apr 20, 2026 - 09:41:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 20.2807iter count: 1.0 +[Apr 20, 2026 - 09:41:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 2852362.0000iter count: 1.0 +[Apr 20, 2026 - 09:41:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6261iter count: 1.0 +[Apr 20, 2026 - 09:41:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.0707 iter count: 1.0 +[Apr 20, 2026 - 09:41:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 3433.5559iter count: 1.0 +[Apr 20, 2026 - 09:41:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 5.0937 iter count: 1.0 +[Apr 20, 2026 - 09:41:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 26566.6582iter count: 1.0 +[Apr 20, 2026 - 09:41:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 6.2141 iter count: 1.0 +[Apr 20, 2026 - 09:41:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 249731.2656iter count: 1.0 +[Apr 20, 2026 - 09:41:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 7.4318 iter count: 1.0 +[Apr 20, 2026 - 09:41:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 2852362.0000iter count: 1.0 +[Apr 20, 2026 - 09:41:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[Apr 20, 2026 - 09:41:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 21.53 seconds +[Apr 20, 2026 - 09:41:46 | INFO | fastgen.trainer:save_checkpoint:266 ] Saving checkpoint iteration 400 +[Apr 20, 2026 - 09:41:46 | INFO | fastgen.utils.checkpointer:save:301 ] Saving FSDP model to prefix /tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000400 +[Apr 20, 2026 - 09:42:03 | SUCCESS | fastgen.utils.checkpointer:save:329 ] Model saved at iteration 400 +[Apr 20, 2026 - 09:42:03 | INFO | fastgen.trainer:validate:358 ] Validation iteration 400 +[Apr 20, 2026 - 09:42:03 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 09:42:07 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 3.3s +[Apr 20, 2026 - 09:42:10 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 09:42:13 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 3.1s +[Apr 20, 2026 - 09:42:17 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 09:42:19 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 2.9s +[Apr 20, 2026 - 09:42:23 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 09:42:26 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 2.9s +[Apr 20, 2026 - 09:42:29 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 09:42:32 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 2.9s +[Apr 20, 2026 - 09:42:35 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 09:42:38 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 2.9s +[Apr 20, 2026 - 09:42:41 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 09:42:44 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 2.9s +[Apr 20, 2026 - 09:42:48 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 09:42:51 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 3.0s +[Apr 20, 2026 - 09:42:54 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 09:42:57 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 2.9s +[Apr 20, 2026 - 09:43:00 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 09:43:03 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 2.9s +[Apr 20, 2026 - 09:43:07 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging val0 stats at iteration 400-------------------- +[Apr 20, 2026 - 09:43:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0000 iter count: 10.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[Apr 20, 2026 - 09:43:23 | INFO | fastgen.callbacks.wandb:on_validation_end:683 ] Logged 10 val videos at iteration 400 +[Apr 20, 2026 - 09:44:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122544 43.234406 42.992031 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.670410 72.554688 69.878906 +util 99.250000 100.000000 98.000000 +[Apr 20, 2026 - 09:44:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 401 : avg iteration time 169.69 seconds +[Apr 20, 2026 - 09:44:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 401 : data loading time 0.00 +[Apr 20, 2026 - 09:44:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 401 : avg forward pass time 13.67 +[Apr 20, 2026 - 09:44:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 401 : backward pass time 12.17 +[Apr 20, 2026 - 09:44:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 401 : optimizer step time 0.98 +[Apr 20, 2026 - 09:44:15 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 401-------------------- +[Apr 20, 2026 - 09:44:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0091 iter count: 1.0 +[Apr 20, 2026 - 09:44:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[Apr 20, 2026 - 09:44:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 09:44:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 09:45:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122656 43.234837 42.992031 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.792480 72.515625 70.210938 +util 96.250000 100.000000 91.000000 +[Apr 20, 2026 - 09:45:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 402 : avg iteration time 52.27 seconds +[Apr 20, 2026 - 09:45:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 402 : data loading time 0.00 +[Apr 20, 2026 - 09:45:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 402 : avg forward pass time 14.01 +[Apr 20, 2026 - 09:45:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 402 : backward pass time 11.56 +[Apr 20, 2026 - 09:45:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 402 : optimizer step time 0.97 +[Apr 20, 2026 - 09:45:07 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 402-------------------- +[Apr 20, 2026 - 09:45:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0083 iter count: 1.0 +[Apr 20, 2026 - 09:45:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[Apr 20, 2026 - 09:45:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 09:45:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 09:46:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122656 43.234837 42.992046 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.636230 72.437500 69.898438 +util 99.000000 100.000000 98.000000 +[Apr 20, 2026 - 09:46:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 403 : avg iteration time 53.73 seconds +[Apr 20, 2026 - 09:46:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 403 : data loading time 0.00 +[Apr 20, 2026 - 09:46:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 403 : avg forward pass time 14.22 +[Apr 20, 2026 - 09:46:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 403 : backward pass time 12.02 +[Apr 20, 2026 - 09:46:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 403 : optimizer step time 1.54 +[Apr 20, 2026 - 09:46:01 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 403-------------------- +[Apr 20, 2026 - 09:46:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0079 iter count: 1.0 +[Apr 20, 2026 - 09:46:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0079 iter count: 1.0 +[Apr 20, 2026 - 09:46:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 09:46:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 09:46:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121902 43.231834 42.992031 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.733887 72.593750 70.035156 +util 97.750000 100.000000 94.000000 +[Apr 20, 2026 - 09:46:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 404 : avg iteration time 52.95 seconds +[Apr 20, 2026 - 09:46:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 404 : data loading time 0.00 +[Apr 20, 2026 - 09:46:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 404 : avg forward pass time 14.29 +[Apr 20, 2026 - 09:46:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 404 : backward pass time 11.74 +[Apr 20, 2026 - 09:46:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 404 : optimizer step time 0.99 +[Apr 20, 2026 - 09:46:54 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 404-------------------- +[Apr 20, 2026 - 09:46:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0088 iter count: 1.0 +[Apr 20, 2026 - 09:46:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[Apr 20, 2026 - 09:46:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 09:46:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 09:50:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122001 43.230881 42.992481 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.541992 135.595703 135.476562 +util 70.500000 74.000000 67.000000 +[Apr 20, 2026 - 09:50:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 405 : avg iteration time 191.80 seconds +[Apr 20, 2026 - 09:50:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 405 : data loading time 0.00 +[Apr 20, 2026 - 09:50:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 405 : avg forward pass time 87.42 +[Apr 20, 2026 - 09:50:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 405 : backward pass time 8.33 +[Apr 20, 2026 - 09:50:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 405 : optimizer step time 0.33 +[Apr 20, 2026 - 09:50:05 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 405-------------------- +[Apr 20, 2026 - 09:50:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 111791.2849 iter count: 1.0 +[Apr 20, 2026 - 09:50:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6370 iter count: 1.0 +[Apr 20, 2026 - 09:50:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 111791.2849iter count: 1.0 +[Apr 20, 2026 - 09:50:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 09:50:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.1850iter count: 1.0 +[Apr 20, 2026 - 09:50:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 2.0805 iter count: 1.0 +[Apr 20, 2026 - 09:50:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.7185 iter count: 1.0 +[Apr 20, 2026 - 09:50:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 197559.7188iter count: 1.0 +[Apr 20, 2026 - 09:50:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 64.1401iter count: 1.0 +[Apr 20, 2026 - 09:50:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 5060305.0000iter count: 1.0 +[Apr 20, 2026 - 09:50:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6370iter count: 1.0 +[Apr 20, 2026 - 09:50:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 2.6783 iter count: 1.0 +[Apr 20, 2026 - 09:50:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 211.9948iter count: 1.0 +[Apr 20, 2026 - 09:50:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 6.1504 iter count: 1.0 +[Apr 20, 2026 - 09:50:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 219889.0938iter count: 1.0 +[Apr 20, 2026 - 09:50:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 3.5694 iter count: 1.0 +[Apr 20, 2026 - 09:50:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 1260.0089iter count: 1.0 +[Apr 20, 2026 - 09:50:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 5.2688 iter count: 1.0 +[Apr 20, 2026 - 09:50:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 37709.2461iter count: 1.0 +[Apr 20, 2026 - 09:50:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0132 iter count: 1.0 +[Apr 20, 2026 - 09:50:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 09:50:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122064 43.231133 42.992481 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.294434 72.105469 69.587891 +util 99.000000 100.000000 96.000000 +[Apr 20, 2026 - 09:50:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 406 : avg iteration time 52.40 seconds +[Apr 20, 2026 - 09:50:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 406 : data loading time 0.00 +[Apr 20, 2026 - 09:50:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 406 : avg forward pass time 13.54 +[Apr 20, 2026 - 09:50:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 406 : backward pass time 11.92 +[Apr 20, 2026 - 09:50:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 406 : optimizer step time 1.54 +[Apr 20, 2026 - 09:50:58 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 406-------------------- +[Apr 20, 2026 - 09:50:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0083 iter count: 1.0 +[Apr 20, 2026 - 09:50:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[Apr 20, 2026 - 09:50:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 09:50:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 09:51:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122064 43.231133 42.992481 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.353027 72.125000 69.742188 +util 95.500000 99.000000 90.000000 +[Apr 20, 2026 - 09:51:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 407 : avg iteration time 58.29 seconds +[Apr 20, 2026 - 09:51:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 407 : data loading time 0.00 +[Apr 20, 2026 - 09:51:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 407 : avg forward pass time 17.08 +[Apr 20, 2026 - 09:51:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 407 : backward pass time 11.54 +[Apr 20, 2026 - 09:51:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 407 : optimizer step time 1.00 +[Apr 20, 2026 - 09:51:56 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 407-------------------- +[Apr 20, 2026 - 09:51:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0089 iter count: 1.0 +[Apr 20, 2026 - 09:51:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[Apr 20, 2026 - 09:51:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 09:51:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 09:52:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122066 43.231136 42.992489 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.674316 69.722656 69.529297 +util 97.750000 99.000000 94.000000 +[Apr 20, 2026 - 09:52:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 408 : avg iteration time 56.30 seconds +[Apr 20, 2026 - 09:52:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 408 : data loading time 0.00 +[Apr 20, 2026 - 09:52:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 408 : avg forward pass time 15.47 +[Apr 20, 2026 - 09:52:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 408 : backward pass time 11.54 +[Apr 20, 2026 - 09:52:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 408 : optimizer step time 1.02 +[Apr 20, 2026 - 09:52:52 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 408-------------------- +[Apr 20, 2026 - 09:52:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0096 iter count: 1.0 +[Apr 20, 2026 - 09:52:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[Apr 20, 2026 - 09:52:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 09:52:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 09:53:51 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122064 43.231133 42.992481 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.309082 72.125000 69.509766 +util 97.500000 100.000000 92.000000 +[Apr 20, 2026 - 09:53:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 409 : avg iteration time 58.67 seconds +[Apr 20, 2026 - 09:53:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 409 : data loading time 0.00 +[Apr 20, 2026 - 09:53:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 409 : avg forward pass time 17.26 +[Apr 20, 2026 - 09:53:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 409 : backward pass time 11.59 +[Apr 20, 2026 - 09:53:51 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 409 : optimizer step time 1.00 +[Apr 20, 2026 - 09:53:51 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 409-------------------- +[Apr 20, 2026 - 09:53:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0091 iter count: 1.0 +[Apr 20, 2026 - 09:53:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0091 iter count: 1.0 +[Apr 20, 2026 - 09:53:51 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 09:53:51 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 09:56:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121766 43.229942 42.992481 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.482422 135.535156 135.382812 +util 67.000000 71.000000 62.000000 +[Apr 20, 2026 - 09:56:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 410 : avg iteration time 182.49 seconds +[Apr 20, 2026 - 09:56:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 410 : data loading time 0.00 +[Apr 20, 2026 - 09:56:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 410 : avg forward pass time 82.74 +[Apr 20, 2026 - 09:56:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 410 : backward pass time 8.38 +[Apr 20, 2026 - 09:56:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 410 : optimizer step time 0.33 +[Apr 20, 2026 - 09:56:54 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 410-------------------- +[Apr 20, 2026 - 09:56:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 46775.2708 iter count: 1.0 +[Apr 20, 2026 - 09:56:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.7158 iter count: 1.0 +[Apr 20, 2026 - 09:56:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 46775.2708iter count: 1.0 +[Apr 20, 2026 - 09:56:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 09:56:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.7056iter count: 1.0 +[Apr 20, 2026 - 09:56:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.0207 iter count: 1.0 +[Apr 20, 2026 - 09:56:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.9401 iter count: 1.0 +[Apr 20, 2026 - 09:56:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 61450.7969iter count: 1.0 +[Apr 20, 2026 - 09:56:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 7.7013 iter count: 1.0 +[Apr 20, 2026 - 09:56:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 1066841.3750iter count: 1.0 +[Apr 20, 2026 - 09:56:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.7158iter count: 1.0 +[Apr 20, 2026 - 09:56:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 2.1575 iter count: 1.0 +[Apr 20, 2026 - 09:56:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 74.8141 iter count: 1.0 +[Apr 20, 2026 - 09:56:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 1.9002 iter count: 1.0 +[Apr 20, 2026 - 09:56:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 44.7207 iter count: 1.0 +[Apr 20, 2026 - 09:56:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 2.7585 iter count: 1.0 +[Apr 20, 2026 - 09:56:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 248.8721iter count: 1.0 +[Apr 20, 2026 - 09:56:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 3.2317 iter count: 1.0 +[Apr 20, 2026 - 09:56:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 641.2302iter count: 1.0 +[Apr 20, 2026 - 09:56:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0080 iter count: 1.0 +[Apr 20, 2026 - 09:56:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 09:57:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121823 43.230171 42.992481 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.933105 70.035156 69.822266 +util 98.500000 100.000000 96.000000 +[Apr 20, 2026 - 09:57:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 411 : avg iteration time 58.24 seconds +[Apr 20, 2026 - 09:57:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 411 : data loading time 0.00 +[Apr 20, 2026 - 09:57:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 411 : avg forward pass time 16.52 +[Apr 20, 2026 - 09:57:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 411 : backward pass time 12.63 +[Apr 20, 2026 - 09:57:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 411 : optimizer step time 1.02 +[Apr 20, 2026 - 09:57:52 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 411-------------------- +[Apr 20, 2026 - 09:57:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0121 iter count: 1.0 +[Apr 20, 2026 - 09:57:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[Apr 20, 2026 - 09:57:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 09:57:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 09:58:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122131 43.231392 42.992493 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.908691 69.996094 69.783203 +util 97.750000 100.000000 91.000000 +[Apr 20, 2026 - 09:58:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 412 : avg iteration time 58.24 seconds +[Apr 20, 2026 - 09:58:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 412 : data loading time 0.00 +[Apr 20, 2026 - 09:58:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 412 : avg forward pass time 16.80 +[Apr 20, 2026 - 09:58:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 412 : backward pass time 11.55 +[Apr 20, 2026 - 09:58:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 412 : optimizer step time 1.00 +[Apr 20, 2026 - 09:58:50 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 412-------------------- +[Apr 20, 2026 - 09:58:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0084 iter count: 1.0 +[Apr 20, 2026 - 09:58:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0084 iter count: 1.0 +[Apr 20, 2026 - 09:58:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 09:58:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 09:59:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122193 43.231640 42.992493 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.903809 69.976562 69.802734 +util 97.500000 100.000000 91.000000 +[Apr 20, 2026 - 09:59:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 413 : avg iteration time 56.79 seconds +[Apr 20, 2026 - 09:59:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 413 : data loading time 0.00 +[Apr 20, 2026 - 09:59:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 413 : avg forward pass time 15.53 +[Apr 20, 2026 - 09:59:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 413 : backward pass time 12.59 +[Apr 20, 2026 - 09:59:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 413 : optimizer step time 1.54 +[Apr 20, 2026 - 09:59:47 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 413-------------------- +[Apr 20, 2026 - 09:59:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0094 iter count: 1.0 +[Apr 20, 2026 - 09:59:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[Apr 20, 2026 - 09:59:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 09:59:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 10:00:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122255 43.231884 42.992496 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.879395 69.976562 69.666016 +util 98.750000 100.000000 97.000000 +[Apr 20, 2026 - 10:00:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 414 : avg iteration time 57.17 seconds +[Apr 20, 2026 - 10:00:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 414 : data loading time 0.00 +[Apr 20, 2026 - 10:00:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 414 : avg forward pass time 15.77 +[Apr 20, 2026 - 10:00:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 414 : backward pass time 12.95 +[Apr 20, 2026 - 10:00:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 414 : optimizer step time 1.12 +[Apr 20, 2026 - 10:00:44 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 414-------------------- +[Apr 20, 2026 - 10:00:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0095 iter count: 1.0 +[Apr 20, 2026 - 10:00:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[Apr 20, 2026 - 10:00:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 10:00:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 10:03:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122037 43.231018 42.992481 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.522461 135.556641 135.498047 +util 65.250000 75.000000 55.000000 +[Apr 20, 2026 - 10:03:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 415 : avg iteration time 180.44 seconds +[Apr 20, 2026 - 10:03:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 415 : data loading time 0.00 +[Apr 20, 2026 - 10:03:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 415 : avg forward pass time 81.46 +[Apr 20, 2026 - 10:03:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 415 : backward pass time 8.84 +[Apr 20, 2026 - 10:03:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 415 : optimizer step time 0.35 +[Apr 20, 2026 - 10:03:44 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 415-------------------- +[Apr 20, 2026 - 10:03:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 169319.8823 iter count: 1.0 +[Apr 20, 2026 - 10:03:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6390 iter count: 1.0 +[Apr 20, 2026 - 10:03:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 169319.8823iter count: 1.0 +[Apr 20, 2026 - 10:03:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 10:03:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.0473iter count: 1.0 +[Apr 20, 2026 - 10:03:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.2995 iter count: 1.0 +[Apr 20, 2026 - 10:03:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.7166 iter count: 1.0 +[Apr 20, 2026 - 10:03:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 201472.2969iter count: 1.0 +[Apr 20, 2026 - 10:03:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 13.4490iter count: 1.0 +[Apr 20, 2026 - 10:03:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 5041471.0000iter count: 1.0 +[Apr 20, 2026 - 10:03:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6390iter count: 1.0 +[Apr 20, 2026 - 10:03:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.4517 iter count: 1.0 +[Apr 20, 2026 - 10:03:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 995.6052iter count: 1.0 +[Apr 20, 2026 - 10:03:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.5882 iter count: 1.0 +[Apr 20, 2026 - 10:03:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 1308.3091iter count: 1.0 +[Apr 20, 2026 - 10:03:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 3.3762 iter count: 1.0 +[Apr 20, 2026 - 10:03:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 856.1791iter count: 1.0 +[Apr 20, 2026 - 10:03:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 1.2995 iter count: 1.0 +[Apr 20, 2026 - 10:03:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 13.4490 iter count: 1.0 +[Apr 20, 2026 - 10:03:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[Apr 20, 2026 - 10:03:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 10:04:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122032 43.230995 42.992481 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.859863 70.015625 69.509766 +util 95.250000 100.000000 92.000000 +[Apr 20, 2026 - 10:04:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 416 : avg iteration time 51.87 seconds +[Apr 20, 2026 - 10:04:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 416 : data loading time 0.00 +[Apr 20, 2026 - 10:04:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 416 : avg forward pass time 13.26 +[Apr 20, 2026 - 10:04:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 416 : backward pass time 11.53 +[Apr 20, 2026 - 10:04:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 416 : optimizer step time 1.00 +[Apr 20, 2026 - 10:04:36 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 416-------------------- +[Apr 20, 2026 - 10:04:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0098 iter count: 1.0 +[Apr 20, 2026 - 10:04:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[Apr 20, 2026 - 10:04:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 10:04:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 10:05:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122029 43.230995 42.992481 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.942871 70.074219 69.587891 +util 99.000000 100.000000 98.000000 +[Apr 20, 2026 - 10:05:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 417 : avg iteration time 65.10 seconds +[Apr 20, 2026 - 10:05:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 417 : data loading time 0.00 +[Apr 20, 2026 - 10:05:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 417 : avg forward pass time 19.75 +[Apr 20, 2026 - 10:05:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 417 : backward pass time 12.96 +[Apr 20, 2026 - 10:05:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 417 : optimizer step time 1.11 +[Apr 20, 2026 - 10:05:41 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 417-------------------- +[Apr 20, 2026 - 10:05:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0130 iter count: 1.0 +[Apr 20, 2026 - 10:05:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0130 iter count: 1.0 +[Apr 20, 2026 - 10:05:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 10:05:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 10:06:41 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122038 43.231007 42.992481 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.879395 70.093750 69.587891 +util 96.250000 99.000000 92.000000 +[Apr 20, 2026 - 10:06:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 418 : avg iteration time 59.94 seconds +[Apr 20, 2026 - 10:06:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 418 : data loading time 0.00 +[Apr 20, 2026 - 10:06:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 418 : avg forward pass time 17.92 +[Apr 20, 2026 - 10:06:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 418 : backward pass time 11.55 +[Apr 20, 2026 - 10:06:41 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 418 : optimizer step time 0.98 +[Apr 20, 2026 - 10:06:41 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 418-------------------- +[Apr 20, 2026 - 10:06:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0135 iter count: 1.0 +[Apr 20, 2026 - 10:06:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0135 iter count: 1.0 +[Apr 20, 2026 - 10:06:41 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 10:06:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 10:07:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122038 43.230999 42.992496 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.874512 70.015625 69.607422 +util 96.000000 100.000000 93.000000 +[Apr 20, 2026 - 10:07:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 419 : avg iteration time 61.96 seconds +[Apr 20, 2026 - 10:07:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 419 : data loading time 0.00 +[Apr 20, 2026 - 10:07:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 419 : avg forward pass time 18.93 +[Apr 20, 2026 - 10:07:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 419 : backward pass time 11.54 +[Apr 20, 2026 - 10:07:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 419 : optimizer step time 1.00 +[Apr 20, 2026 - 10:07:43 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 419-------------------- +[Apr 20, 2026 - 10:07:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0096 iter count: 1.0 +[Apr 20, 2026 - 10:07:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[Apr 20, 2026 - 10:07:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 10:07:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 10:10:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121784 43.229992 42.992504 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.527344 135.576172 135.439453 +util 75.000000 95.000000 64.000000 +[Apr 20, 2026 - 10:10:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 420 : avg iteration time 189.43 seconds +[Apr 20, 2026 - 10:10:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 420 : data loading time 0.00 +[Apr 20, 2026 - 10:10:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 420 : avg forward pass time 85.94 +[Apr 20, 2026 - 10:10:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 420 : backward pass time 8.92 +[Apr 20, 2026 - 10:10:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 420 : optimizer step time 0.34 +[Apr 20, 2026 - 10:10:53 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 420-------------------- +[Apr 20, 2026 - 10:10:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 43555.5527 iter count: 1.0 +[Apr 20, 2026 - 10:10:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6334 iter count: 1.0 +[Apr 20, 2026 - 10:10:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 43555.5527iter count: 1.0 +[Apr 20, 2026 - 10:10:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 10:10:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.4969iter count: 1.0 +[Apr 20, 2026 - 10:10:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.9919 iter count: 1.0 +[Apr 20, 2026 - 10:10:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.9246 iter count: 1.0 +[Apr 20, 2026 - 10:10:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 78795.8203iter count: 1.0 +[Apr 20, 2026 - 10:10:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 53.7199iter count: 1.0 +[Apr 20, 2026 - 10:10:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 1034324.7500iter count: 1.0 +[Apr 20, 2026 - 10:10:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6334iter count: 1.0 +[Apr 20, 2026 - 10:10:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 5.8711 iter count: 1.0 +[Apr 20, 2026 - 10:10:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 125765.4766iter count: 1.0 +[Apr 20, 2026 - 10:10:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.6064 iter count: 1.0 +[Apr 20, 2026 - 10:10:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 10024.0059iter count: 1.0 +[Apr 20, 2026 - 10:10:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.9918 iter count: 1.0 +[Apr 20, 2026 - 10:10:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 21669.0176iter count: 1.0 +[Apr 20, 2026 - 10:10:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 5.0922 iter count: 1.0 +[Apr 20, 2026 - 10:10:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 26486.9707iter count: 1.0 +[Apr 20, 2026 - 10:10:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0078 iter count: 1.0 +[Apr 20, 2026 - 10:10:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 10:11:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121958 43.230709 42.992481 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.645996 72.710938 69.880859 +util 98.500000 100.000000 95.000000 +[Apr 20, 2026 - 10:11:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 421 : avg iteration time 64.44 seconds +[Apr 20, 2026 - 10:11:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 421 : data loading time 0.00 +[Apr 20, 2026 - 10:11:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 421 : avg forward pass time 20.17 +[Apr 20, 2026 - 10:11:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 421 : backward pass time 11.54 +[Apr 20, 2026 - 10:11:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 421 : optimizer step time 0.98 +[Apr 20, 2026 - 10:11:57 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 421-------------------- +[Apr 20, 2026 - 10:11:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0127 iter count: 1.0 +[Apr 20, 2026 - 10:11:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0127 iter count: 1.0 +[Apr 20, 2026 - 10:11:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 10:11:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 10:12:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122199 43.231674 42.992481 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.923340 69.996094 69.822266 +util 98.000000 99.000000 95.000000 +[Apr 20, 2026 - 10:12:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 422 : avg iteration time 62.00 seconds +[Apr 20, 2026 - 10:12:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 422 : data loading time 0.00 +[Apr 20, 2026 - 10:12:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 422 : avg forward pass time 18.96 +[Apr 20, 2026 - 10:12:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 422 : backward pass time 11.55 +[Apr 20, 2026 - 10:12:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 422 : optimizer step time 1.00 +[Apr 20, 2026 - 10:12:59 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 422-------------------- +[Apr 20, 2026 - 10:12:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0105 iter count: 1.0 +[Apr 20, 2026 - 10:12:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[Apr 20, 2026 - 10:12:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 10:12:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 10:13:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122205 43.231674 42.992500 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.650879 72.691406 69.841797 +util 97.250000 99.000000 92.000000 +[Apr 20, 2026 - 10:13:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 423 : avg iteration time 58.86 seconds +[Apr 20, 2026 - 10:13:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 423 : data loading time 0.00 +[Apr 20, 2026 - 10:13:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 423 : avg forward pass time 16.63 +[Apr 20, 2026 - 10:13:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 423 : backward pass time 11.53 +[Apr 20, 2026 - 10:13:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 423 : optimizer step time 1.02 +[Apr 20, 2026 - 10:13:58 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 423-------------------- +[Apr 20, 2026 - 10:13:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0094 iter count: 1.0 +[Apr 20, 2026 - 10:13:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[Apr 20, 2026 - 10:13:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 10:13:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 10:14:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122241 43.231842 42.992481 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.616699 72.691406 69.822266 +util 97.250000 99.000000 94.000000 +[Apr 20, 2026 - 10:14:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 424 : avg iteration time 58.52 seconds +[Apr 20, 2026 - 10:14:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 424 : data loading time 0.00 +[Apr 20, 2026 - 10:14:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 424 : avg forward pass time 17.20 +[Apr 20, 2026 - 10:14:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 424 : backward pass time 11.52 +[Apr 20, 2026 - 10:14:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 424 : optimizer step time 1.02 +[Apr 20, 2026 - 10:14:57 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 424-------------------- +[Apr 20, 2026 - 10:14:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0098 iter count: 1.0 +[Apr 20, 2026 - 10:14:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[Apr 20, 2026 - 10:14:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 10:14:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 10:17:58 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121970 43.230743 42.992496 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.483398 135.537109 135.458984 +util 67.500000 81.000000 50.000000 +[Apr 20, 2026 - 10:17:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 425 : avg iteration time 181.37 seconds +[Apr 20, 2026 - 10:17:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 425 : data loading time 0.00 +[Apr 20, 2026 - 10:17:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 425 : avg forward pass time 81.90 +[Apr 20, 2026 - 10:17:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 425 : backward pass time 8.89 +[Apr 20, 2026 - 10:17:58 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 425 : optimizer step time 0.34 +[Apr 20, 2026 - 10:17:58 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 425-------------------- +[Apr 20, 2026 - 10:17:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 20510.9782 iter count: 1.0 +[Apr 20, 2026 - 10:17:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5860 iter count: 1.0 +[Apr 20, 2026 - 10:17:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 20510.9782iter count: 1.0 +[Apr 20, 2026 - 10:17:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 10:17:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.7209iter count: 1.0 +[Apr 20, 2026 - 10:17:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.6778 iter count: 1.0 +[Apr 20, 2026 - 10:17:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.8224 iter count: 1.0 +[Apr 20, 2026 - 10:17:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 39441.0469iter count: 1.0 +[Apr 20, 2026 - 10:17:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 3.8794 iter count: 1.0 +[Apr 20, 2026 - 10:17:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 843142.8750iter count: 1.0 +[Apr 20, 2026 - 10:17:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5860iter count: 1.0 +[Apr 20, 2026 - 10:17:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.8038 iter count: 1.0 +[Apr 20, 2026 - 10:17:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 2013.5499iter count: 1.0 +[Apr 20, 2026 - 10:17:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.0865 iter count: 1.0 +[Apr 20, 2026 - 10:17:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 3544.0627iter count: 1.0 +[Apr 20, 2026 - 10:17:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 2.2419 iter count: 1.0 +[Apr 20, 2026 - 10:17:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 88.5680 iter count: 1.0 +[Apr 20, 2026 - 10:17:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 6.8224 iter count: 1.0 +[Apr 20, 2026 - 10:17:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 843142.8750iter count: 1.0 +[Apr 20, 2026 - 10:17:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0075 iter count: 1.0 +[Apr 20, 2026 - 10:17:58 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 10:18:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122037 43.231010 42.992496 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.986816 70.035156 69.861328 +util 98.750000 99.000000 98.000000 +[Apr 20, 2026 - 10:18:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 426 : avg iteration time 53.88 seconds +[Apr 20, 2026 - 10:18:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 426 : data loading time 0.00 +[Apr 20, 2026 - 10:18:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 426 : avg forward pass time 14.62 +[Apr 20, 2026 - 10:18:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 426 : backward pass time 11.55 +[Apr 20, 2026 - 10:18:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 426 : optimizer step time 1.08 +[Apr 20, 2026 - 10:18:52 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 426-------------------- +[Apr 20, 2026 - 10:18:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0105 iter count: 1.0 +[Apr 20, 2026 - 10:18:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[Apr 20, 2026 - 10:18:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 10:18:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 10:19:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122027 43.230988 42.992481 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.748535 72.691406 69.802734 +util 98.000000 100.000000 93.000000 +[Apr 20, 2026 - 10:19:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 427 : avg iteration time 55.45 seconds +[Apr 20, 2026 - 10:19:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 427 : data loading time 0.00 +[Apr 20, 2026 - 10:19:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 427 : avg forward pass time 15.65 +[Apr 20, 2026 - 10:19:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 427 : backward pass time 11.54 +[Apr 20, 2026 - 10:19:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 427 : optimizer step time 0.99 +[Apr 20, 2026 - 10:19:47 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 427-------------------- +[Apr 20, 2026 - 10:19:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0086 iter count: 1.0 +[Apr 20, 2026 - 10:19:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[Apr 20, 2026 - 10:19:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 10:19:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 10:20:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122040 43.231010 42.992504 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.738770 72.652344 69.802734 +util 95.250000 100.000000 92.000000 +[Apr 20, 2026 - 10:20:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 428 : avg iteration time 58.84 seconds +[Apr 20, 2026 - 10:20:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 428 : data loading time 0.00 +[Apr 20, 2026 - 10:20:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 428 : avg forward pass time 16.82 +[Apr 20, 2026 - 10:20:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 428 : backward pass time 11.55 +[Apr 20, 2026 - 10:20:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 428 : optimizer step time 1.01 +[Apr 20, 2026 - 10:20:46 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 428-------------------- +[Apr 20, 2026 - 10:20:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0115 iter count: 1.0 +[Apr 20, 2026 - 10:20:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[Apr 20, 2026 - 10:20:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 10:20:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 10:21:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122037 43.230988 42.992496 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.753418 72.710938 69.802734 +util 96.750000 100.000000 93.000000 +[Apr 20, 2026 - 10:21:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 429 : avg iteration time 56.26 seconds +[Apr 20, 2026 - 10:21:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 429 : data loading time 0.00 +[Apr 20, 2026 - 10:21:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 429 : avg forward pass time 16.05 +[Apr 20, 2026 - 10:21:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 429 : backward pass time 11.61 +[Apr 20, 2026 - 10:21:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 429 : optimizer step time 0.97 +[Apr 20, 2026 - 10:21:42 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 429-------------------- +[Apr 20, 2026 - 10:21:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0083 iter count: 1.0 +[Apr 20, 2026 - 10:21:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[Apr 20, 2026 - 10:21:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 10:21:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 10:24:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121658 43.229492 42.992481 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 135.546875 135.615234 135.441406 +util 69.500000 73.000000 65.000000 +[Apr 20, 2026 - 10:24:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 430 : avg iteration time 189.65 seconds +[Apr 20, 2026 - 10:24:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 430 : data loading time 0.00 +[Apr 20, 2026 - 10:24:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 430 : avg forward pass time 86.26 +[Apr 20, 2026 - 10:24:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 430 : backward pass time 8.42 +[Apr 20, 2026 - 10:24:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 430 : optimizer step time 0.36 +[Apr 20, 2026 - 10:24:52 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 430-------------------- +[Apr 20, 2026 - 10:24:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 96768.2588 iter count: 1.0 +[Apr 20, 2026 - 10:24:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5803 iter count: 1.0 +[Apr 20, 2026 - 10:24:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 96768.2588iter count: 1.0 +[Apr 20, 2026 - 10:24:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 10:24:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.4213iter count: 1.0 +[Apr 20, 2026 - 10:24:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.1770 iter count: 1.0 +[Apr 20, 2026 - 10:24:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.3343 iter count: 1.0 +[Apr 20, 2026 - 10:24:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 184777.5625iter count: 1.0 +[Apr 20, 2026 - 10:24:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 10.5283iter count: 1.0 +[Apr 20, 2026 - 10:24:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 2346720.7500iter count: 1.0 +[Apr 20, 2026 - 10:24:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5803iter count: 1.0 +[Apr 20, 2026 - 10:24:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.9198 iter count: 1.0 +[Apr 20, 2026 - 10:24:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 2539.4253iter count: 1.0 +[Apr 20, 2026 - 10:24:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.2774 iter count: 1.0 +[Apr 20, 2026 - 10:24:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 5191.5356iter count: 1.0 +[Apr 20, 2026 - 10:24:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.5859 iter count: 1.0 +[Apr 20, 2026 - 10:24:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 9622.6318iter count: 1.0 +[Apr 20, 2026 - 10:24:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 1.4642 iter count: 1.0 +[Apr 20, 2026 - 10:24:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 18.6992 iter count: 1.0 +[Apr 20, 2026 - 10:24:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[Apr 20, 2026 - 10:24:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 10:25:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121721 43.229740 42.992504 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.874512 70.054688 69.529297 +util 95.250000 100.000000 91.000000 +[Apr 20, 2026 - 10:25:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 431 : avg iteration time 63.15 seconds +[Apr 20, 2026 - 10:25:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 431 : data loading time 0.00 +[Apr 20, 2026 - 10:25:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 431 : avg forward pass time 19.49 +[Apr 20, 2026 - 10:25:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 431 : backward pass time 11.59 +[Apr 20, 2026 - 10:25:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 431 : optimizer step time 0.99 +[Apr 20, 2026 - 10:25:55 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 431-------------------- +[Apr 20, 2026 - 10:25:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0113 iter count: 1.0 +[Apr 20, 2026 - 10:25:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[Apr 20, 2026 - 10:25:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 10:25:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 10:26:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122089 43.231236 42.992481 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.928223 70.015625 69.685547 +util 96.000000 99.000000 91.000000 +[Apr 20, 2026 - 10:26:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 432 : avg iteration time 59.76 seconds +[Apr 20, 2026 - 10:26:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 432 : data loading time 0.00 +[Apr 20, 2026 - 10:26:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 432 : avg forward pass time 17.29 +[Apr 20, 2026 - 10:26:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 432 : backward pass time 11.83 +[Apr 20, 2026 - 10:26:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 432 : optimizer step time 1.52 +[Apr 20, 2026 - 10:26:55 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 432-------------------- +[Apr 20, 2026 - 10:26:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0103 iter count: 1.0 +[Apr 20, 2026 - 10:26:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[Apr 20, 2026 - 10:26:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 10:26:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 10:27:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122095 43.231236 42.992504 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.806152 69.996094 69.509766 +util 96.250000 99.000000 94.000000 +[Apr 20, 2026 - 10:27:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 433 : avg iteration time 52.46 seconds +[Apr 20, 2026 - 10:27:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 433 : data loading time 0.00 +[Apr 20, 2026 - 10:27:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 433 : avg forward pass time 13.53 +[Apr 20, 2026 - 10:27:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 433 : backward pass time 12.68 +[Apr 20, 2026 - 10:27:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 433 : optimizer step time 1.05 +[Apr 20, 2026 - 10:27:47 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 433-------------------- +[Apr 20, 2026 - 10:27:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0096 iter count: 1.0 +[Apr 20, 2026 - 10:27:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[Apr 20, 2026 - 10:27:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 10:27:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 10:28:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122042 43.231026 42.992500 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.889160 70.015625 69.822266 +util 98.500000 99.000000 97.000000 +[Apr 20, 2026 - 10:28:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 434 : avg iteration time 58.46 seconds +[Apr 20, 2026 - 10:28:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 434 : data loading time 0.00 +[Apr 20, 2026 - 10:28:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 434 : avg forward pass time 17.00 +[Apr 20, 2026 - 10:28:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 434 : backward pass time 11.66 +[Apr 20, 2026 - 10:28:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 434 : optimizer step time 0.99 +[Apr 20, 2026 - 10:28:46 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 434-------------------- +[Apr 20, 2026 - 10:28:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0102 iter count: 1.0 +[Apr 20, 2026 - 10:28:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[Apr 20, 2026 - 10:28:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 10:28:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 10:32:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121861 43.23032 42.992481 +peak_gpu_mem_gb 131.547835 131.54938 131.547320 +peak_gpu_mem_reserved_gb 135.531738 135.59375 135.421875 +util 72.250000 80.00000 66.000000 +[Apr 20, 2026 - 10:32:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 435 : avg iteration time 198.71 seconds +[Apr 20, 2026 - 10:32:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 435 : data loading time 0.00 +[Apr 20, 2026 - 10:32:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 435 : avg forward pass time 90.83 +[Apr 20, 2026 - 10:32:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 435 : backward pass time 8.41 +[Apr 20, 2026 - 10:32:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 435 : optimizer step time 0.34 +[Apr 20, 2026 - 10:32:05 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 435-------------------- +[Apr 20, 2026 - 10:32:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 387886.8975 iter count: 1.0 +[Apr 20, 2026 - 10:32:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5952 iter count: 1.0 +[Apr 20, 2026 - 10:32:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 387886.8975iter count: 1.0 +[Apr 20, 2026 - 10:32:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 10:32:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.4596iter count: 1.0 +[Apr 20, 2026 - 10:32:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.2614 iter count: 1.0 +[Apr 20, 2026 - 10:32:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 8.0886 iter count: 1.0 +[Apr 20, 2026 - 10:32:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 519640.1250iter count: 1.0 +[Apr 20, 2026 - 10:32:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 12.4635iter count: 1.0 +[Apr 20, 2026 - 10:32:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 10609631.0000iter count: 1.0 +[Apr 20, 2026 - 10:32:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5952iter count: 1.0 +[Apr 20, 2026 - 10:32:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.5621 iter count: 1.0 +[Apr 20, 2026 - 10:32:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 1241.6744iter count: 1.0 +[Apr 20, 2026 - 10:32:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 1.9804 iter count: 1.0 +[Apr 20, 2026 - 10:32:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 52.4983 iter count: 1.0 +[Apr 20, 2026 - 10:32:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 6.1492 iter count: 1.0 +[Apr 20, 2026 - 10:32:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 219324.0469iter count: 1.0 +[Apr 20, 2026 - 10:32:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 2.7428 iter count: 1.0 +[Apr 20, 2026 - 10:32:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 241.1875iter count: 1.0 +[Apr 20, 2026 - 10:32:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[Apr 20, 2026 - 10:32:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 10:33:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122046 43.231064 42.992481 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.499512 72.750000 69.412109 +util 97.500000 100.000000 93.000000 +[Apr 20, 2026 - 10:33:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 436 : avg iteration time 61.05 seconds +[Apr 20, 2026 - 10:33:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 436 : data loading time 0.00 +[Apr 20, 2026 - 10:33:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 436 : avg forward pass time 17.45 +[Apr 20, 2026 - 10:33:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 436 : backward pass time 11.84 +[Apr 20, 2026 - 10:33:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 436 : optimizer step time 1.49 +[Apr 20, 2026 - 10:33:06 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 436-------------------- +[Apr 20, 2026 - 10:33:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0103 iter count: 1.0 +[Apr 20, 2026 - 10:33:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[Apr 20, 2026 - 10:33:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 10:33:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 10:34:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122115 43.231316 42.992481 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.523926 72.476562 69.548828 +util 95.500000 100.000000 92.000000 +[Apr 20, 2026 - 10:34:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 437 : avg iteration time 61.35 seconds +[Apr 20, 2026 - 10:34:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 437 : data loading time 0.00 +[Apr 20, 2026 - 10:34:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 437 : avg forward pass time 18.62 +[Apr 20, 2026 - 10:34:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 437 : backward pass time 11.56 +[Apr 20, 2026 - 10:34:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 437 : optimizer step time 0.97 +[Apr 20, 2026 - 10:34:07 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 437-------------------- +[Apr 20, 2026 - 10:34:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0108 iter count: 1.0 +[Apr 20, 2026 - 10:34:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[Apr 20, 2026 - 10:34:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 10:34:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 10:35:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122115 43.231316 42.992504 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.869629 70.015625 69.548828 +util 94.250000 99.000000 91.000000 +[Apr 20, 2026 - 10:35:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 438 : avg iteration time 62.79 seconds +[Apr 20, 2026 - 10:35:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 438 : data loading time 0.00 +[Apr 20, 2026 - 10:35:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 438 : avg forward pass time 18.63 +[Apr 20, 2026 - 10:35:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 438 : backward pass time 11.55 +[Apr 20, 2026 - 10:35:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 438 : optimizer step time 1.02 +[Apr 20, 2026 - 10:35:10 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 438-------------------- +[Apr 20, 2026 - 10:35:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0092 iter count: 1.0 +[Apr 20, 2026 - 10:35:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[Apr 20, 2026 - 10:35:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 10:35:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 10:36:11 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122114 43.231316 42.992481 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.484863 72.632812 69.548828 +util 98.000000 100.000000 93.000000 +[Apr 20, 2026 - 10:36:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 439 : avg iteration time 60.97 seconds +[Apr 20, 2026 - 10:36:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 439 : data loading time 0.00 +[Apr 20, 2026 - 10:36:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 439 : avg forward pass time 18.43 +[Apr 20, 2026 - 10:36:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 439 : backward pass time 11.55 +[Apr 20, 2026 - 10:36:11 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 439 : optimizer step time 0.99 +[Apr 20, 2026 - 10:36:11 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 439-------------------- +[Apr 20, 2026 - 10:36:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0100 iter count: 1.0 +[Apr 20, 2026 - 10:36:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[Apr 20, 2026 - 10:36:11 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 10:36:11 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 10:39:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121675 43.229561 42.992481 +peak_gpu_mem_gb 131.547835 131.549380 131.547320 +peak_gpu_mem_reserved_gb 134.829102 135.576172 132.937500 +util 83.250000 99.000000 75.000000 +[Apr 20, 2026 - 10:39:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 440 : avg iteration time 187.82 seconds +[Apr 20, 2026 - 10:39:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 440 : data loading time 0.00 +[Apr 20, 2026 - 10:39:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 440 : avg forward pass time 83.41 +[Apr 20, 2026 - 10:39:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 440 : backward pass time 12.31 +[Apr 20, 2026 - 10:39:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 440 : optimizer step time 0.34 +[Apr 20, 2026 - 10:39:19 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 440-------------------- +[Apr 20, 2026 - 10:39:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 203053.9088 iter count: 1.0 +[Apr 20, 2026 - 10:39:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.7015 iter count: 1.0 +[Apr 20, 2026 - 10:39:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 203053.9088iter count: 1.0 +[Apr 20, 2026 - 10:39:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 10:39:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.6279iter count: 1.0 +[Apr 20, 2026 - 10:39:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.0131 iter count: 1.0 +[Apr 20, 2026 - 10:39:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 8.1697 iter count: 1.0 +[Apr 20, 2026 - 10:39:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 399038.4062iter count: 1.0 +[Apr 20, 2026 - 10:39:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 7.5851 iter count: 1.0 +[Apr 20, 2026 - 10:39:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 12477427.0000iter count: 1.0 +[Apr 20, 2026 - 10:39:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.7015iter count: 1.0 +[Apr 20, 2026 - 10:39:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 5.2591 iter count: 1.0 +[Apr 20, 2026 - 10:39:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 36979.9531iter count: 1.0 +[Apr 20, 2026 - 10:39:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.2809 iter count: 1.0 +[Apr 20, 2026 - 10:39:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 707.5751iter count: 1.0 +[Apr 20, 2026 - 10:39:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 3.4625 iter count: 1.0 +[Apr 20, 2026 - 10:39:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 1017.3663iter count: 1.0 +[Apr 20, 2026 - 10:39:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.2884 iter count: 1.0 +[Apr 20, 2026 - 10:39:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 5307.4565iter count: 1.0 +[Apr 20, 2026 - 10:39:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0083 iter count: 1.0 +[Apr 20, 2026 - 10:39:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 10:40:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121918 43.230549 42.992481 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.626465 72.632812 69.841797 +util 97.750000 100.000000 93.000000 +[Apr 20, 2026 - 10:40:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 441 : avg iteration time 74.59 seconds +[Apr 20, 2026 - 10:40:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 441 : data loading time 0.00 +[Apr 20, 2026 - 10:40:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 441 : avg forward pass time 21.71 +[Apr 20, 2026 - 10:40:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 441 : backward pass time 15.28 +[Apr 20, 2026 - 10:40:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 441 : optimizer step time 1.73 +[Apr 20, 2026 - 10:40:33 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 441-------------------- +[Apr 20, 2026 - 10:40:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0081 iter count: 1.0 +[Apr 20, 2026 - 10:40:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0081 iter count: 1.0 +[Apr 20, 2026 - 10:40:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 10:40:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 10:41:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121920 43.230549 42.992481 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.523926 72.515625 69.666016 +util 98.000000 100.000000 94.000000 +[Apr 20, 2026 - 10:41:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 442 : avg iteration time 61.85 seconds +[Apr 20, 2026 - 10:41:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 442 : data loading time 0.00 +[Apr 20, 2026 - 10:41:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 442 : avg forward pass time 15.16 +[Apr 20, 2026 - 10:41:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 442 : backward pass time 14.23 +[Apr 20, 2026 - 10:41:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 442 : optimizer step time 1.73 +[Apr 20, 2026 - 10:41:35 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 442-------------------- +[Apr 20, 2026 - 10:41:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0101 iter count: 1.0 +[Apr 20, 2026 - 10:41:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[Apr 20, 2026 - 10:41:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 10:41:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 10:42:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121923 43.230549 42.992504 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.533691 72.515625 69.822266 +util 98.500000 100.000000 94.000000 +[Apr 20, 2026 - 10:42:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 443 : avg iteration time 80.38 seconds +[Apr 20, 2026 - 10:42:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 443 : data loading time 0.00 +[Apr 20, 2026 - 10:42:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 443 : avg forward pass time 25.09 +[Apr 20, 2026 - 10:42:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 443 : backward pass time 14.25 +[Apr 20, 2026 - 10:42:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 443 : optimizer step time 1.75 +[Apr 20, 2026 - 10:42:55 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 443-------------------- +[Apr 20, 2026 - 10:42:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0096 iter count: 1.0 +[Apr 20, 2026 - 10:42:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[Apr 20, 2026 - 10:42:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 10:42:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 10:44:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.121931 43.230572 42.992481 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.489746 72.515625 69.646484 +util 99.500000 100.000000 98.000000 +[Apr 20, 2026 - 10:44:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 444 : avg iteration time 69.31 seconds +[Apr 20, 2026 - 10:44:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 444 : data loading time 0.00 +[Apr 20, 2026 - 10:44:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 444 : avg forward pass time 18.48 +[Apr 20, 2026 - 10:44:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 444 : backward pass time 15.31 +[Apr 20, 2026 - 10:44:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 444 : optimizer step time 1.73 +[Apr 20, 2026 - 10:44:05 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 444-------------------- +[Apr 20, 2026 - 10:44:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0098 iter count: 1.0 +[Apr 20, 2026 - 10:44:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[Apr 20, 2026 - 10:44:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 10:44:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 10:48:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122469 43.231163 42.992496 +peak_gpu_mem_gb 129.547819 131.549380 123.547254 +peak_gpu_mem_reserved_gb 133.511230 135.595703 127.568359 +util 83.000000 100.000000 73.000000 +[Apr 20, 2026 - 10:48:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 445 : avg iteration time 262.08 seconds +[Apr 20, 2026 - 10:48:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 445 : data loading time 0.00 +[Apr 20, 2026 - 10:48:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 445 : avg forward pass time 117.76 +[Apr 20, 2026 - 10:48:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 445 : backward pass time 11.48 +[Apr 20, 2026 - 10:48:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 445 : optimizer step time 0.36 +[Apr 20, 2026 - 10:48:27 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 445-------------------- +[Apr 20, 2026 - 10:48:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 229368.2164 iter count: 1.0 +[Apr 20, 2026 - 10:48:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6552 iter count: 1.0 +[Apr 20, 2026 - 10:48:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 229368.2164iter count: 1.0 +[Apr 20, 2026 - 10:48:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 10:48:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.4680iter count: 1.0 +[Apr 20, 2026 - 10:48:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.7776 iter count: 1.0 +[Apr 20, 2026 - 10:48:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.8408 iter count: 1.0 +[Apr 20, 2026 - 10:48:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 392394.6250iter count: 1.0 +[Apr 20, 2026 - 10:48:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 4.7359 iter count: 1.0 +[Apr 20, 2026 - 10:48:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 6463377.5000iter count: 1.0 +[Apr 20, 2026 - 10:48:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6552iter count: 1.0 +[Apr 20, 2026 - 10:48:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.6179 iter count: 1.0 +[Apr 20, 2026 - 10:48:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 10257.9805iter count: 1.0 +[Apr 20, 2026 - 10:48:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.9139 iter count: 1.0 +[Apr 20, 2026 - 10:48:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 18541.7051iter count: 1.0 +[Apr 20, 2026 - 10:48:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 2.6621 iter count: 1.0 +[Apr 20, 2026 - 10:48:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 205.2249iter count: 1.0 +[Apr 20, 2026 - 10:48:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 5.5081 iter count: 1.0 +[Apr 20, 2026 - 10:48:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 60846.9570iter count: 1.0 +[Apr 20, 2026 - 10:48:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[Apr 20, 2026 - 10:48:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 10:49:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122595 43.231163 42.992512 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.611816 72.515625 69.822266 +util 98.500000 100.000000 95.000000 +[Apr 20, 2026 - 10:49:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 446 : avg iteration time 83.41 seconds +[Apr 20, 2026 - 10:49:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 446 : data loading time 0.00 +[Apr 20, 2026 - 10:49:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 446 : avg forward pass time 26.21 +[Apr 20, 2026 - 10:49:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 446 : backward pass time 14.23 +[Apr 20, 2026 - 10:49:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 446 : optimizer step time 2.59 +[Apr 20, 2026 - 10:49:50 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 446-------------------- +[Apr 20, 2026 - 10:49:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0102 iter count: 1.0 +[Apr 20, 2026 - 10:49:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[Apr 20, 2026 - 10:49:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 10:49:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.05 seconds +[Apr 20, 2026 - 10:50:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122843 43.231316 42.992481 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.928223 70.093750 69.646484 +util 97.000000 99.000000 91.000000 +[Apr 20, 2026 - 10:50:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 447 : avg iteration time 62.18 seconds +[Apr 20, 2026 - 10:50:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 447 : data loading time 0.00 +[Apr 20, 2026 - 10:50:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 447 : avg forward pass time 15.95 +[Apr 20, 2026 - 10:50:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 447 : backward pass time 13.18 +[Apr 20, 2026 - 10:50:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 447 : optimizer step time 1.07 +[Apr 20, 2026 - 10:50:52 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 447-------------------- +[Apr 20, 2026 - 10:50:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0079 iter count: 1.0 +[Apr 20, 2026 - 10:50:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0079 iter count: 1.0 +[Apr 20, 2026 - 10:50:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 10:50:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 10:51:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122847 43.231316 42.992485 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.562988 72.554688 69.626953 +util 94.750000 99.000000 91.000000 +[Apr 20, 2026 - 10:51:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 448 : avg iteration time 57.13 seconds +[Apr 20, 2026 - 10:51:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 448 : data loading time 0.00 +[Apr 20, 2026 - 10:51:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 448 : avg forward pass time 15.67 +[Apr 20, 2026 - 10:51:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 448 : backward pass time 12.96 +[Apr 20, 2026 - 10:51:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 448 : optimizer step time 1.23 +[Apr 20, 2026 - 10:51:49 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 448-------------------- +[Apr 20, 2026 - 10:51:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0077 iter count: 1.0 +[Apr 20, 2026 - 10:51:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0077 iter count: 1.0 +[Apr 20, 2026 - 10:51:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 10:51:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 10:52:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122850 43.231354 42.992485 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.933105 70.035156 69.626953 +util 96.500000 100.000000 92.000000 +[Apr 20, 2026 - 10:52:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 449 : avg iteration time 52.89 seconds +[Apr 20, 2026 - 10:52:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 449 : data loading time 0.00 +[Apr 20, 2026 - 10:52:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 449 : avg forward pass time 14.37 +[Apr 20, 2026 - 10:52:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 449 : backward pass time 11.57 +[Apr 20, 2026 - 10:52:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 449 : optimizer step time 1.00 +[Apr 20, 2026 - 10:52:42 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 449-------------------- +[Apr 20, 2026 - 10:52:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0079 iter count: 1.0 +[Apr 20, 2026 - 10:52:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0079 iter count: 1.0 +[Apr 20, 2026 - 10:52:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 10:52:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 10:55:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122695 43.231171 42.992485 +peak_gpu_mem_gb 129.547819 131.549380 123.547254 +peak_gpu_mem_reserved_gb 133.525879 135.558594 127.570312 +util 69.750000 73.000000 66.000000 +[Apr 20, 2026 - 10:55:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 450 : avg iteration time 189.70 seconds +[Apr 20, 2026 - 10:55:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 450 : data loading time 0.00 +[Apr 20, 2026 - 10:55:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 450 : avg forward pass time 86.27 +[Apr 20, 2026 - 10:55:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 450 : backward pass time 8.48 +[Apr 20, 2026 - 10:55:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 450 : optimizer step time 0.34 +[Apr 20, 2026 - 10:55:52 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 450-------------------- +[Apr 20, 2026 - 10:55:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 80968.0015 iter count: 1.0 +[Apr 20, 2026 - 10:55:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6716 iter count: 1.0 +[Apr 20, 2026 - 10:55:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 80968.0015iter count: 1.0 +[Apr 20, 2026 - 10:55:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 10:55:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.4058iter count: 1.0 +[Apr 20, 2026 - 10:55:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.7622 iter count: 1.0 +[Apr 20, 2026 - 10:55:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.0872 iter count: 1.0 +[Apr 20, 2026 - 10:55:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 99913.2656iter count: 1.0 +[Apr 20, 2026 - 10:55:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 4.5927 iter count: 1.0 +[Apr 20, 2026 - 10:55:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 1431614.7500iter count: 1.0 +[Apr 20, 2026 - 10:55:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6716iter count: 1.0 +[Apr 20, 2026 - 10:55:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.4549 iter count: 1.0 +[Apr 20, 2026 - 10:55:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 1002.1209iter count: 1.0 +[Apr 20, 2026 - 10:55:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.0657 iter count: 1.0 +[Apr 20, 2026 - 10:55:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 460.1022iter count: 1.0 +[Apr 20, 2026 - 10:55:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.9568 iter count: 1.0 +[Apr 20, 2026 - 10:55:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 20204.2559iter count: 1.0 +[Apr 20, 2026 - 10:55:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.1351 iter count: 1.0 +[Apr 20, 2026 - 10:55:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 3905.9265iter count: 1.0 +[Apr 20, 2026 - 10:55:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[Apr 20, 2026 - 10:55:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 10:56:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122692 43.231178 42.992485 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.519531 72.478516 69.587891 +util 98.750000 100.000000 97.000000 +[Apr 20, 2026 - 10:56:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 451 : avg iteration time 60.42 seconds +[Apr 20, 2026 - 10:56:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 451 : data loading time 0.00 +[Apr 20, 2026 - 10:56:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 451 : avg forward pass time 17.56 +[Apr 20, 2026 - 10:56:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 451 : backward pass time 11.62 +[Apr 20, 2026 - 10:56:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 451 : optimizer step time 0.97 +[Apr 20, 2026 - 10:56:52 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 451-------------------- +[Apr 20, 2026 - 10:56:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0086 iter count: 1.0 +[Apr 20, 2026 - 10:56:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[Apr 20, 2026 - 10:56:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 10:56:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 10:57:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122694 43.231163 42.992485 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.543945 72.673828 69.568359 +util 97.500000 100.000000 94.000000 +[Apr 20, 2026 - 10:57:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 452 : avg iteration time 56.90 seconds +[Apr 20, 2026 - 10:57:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 452 : data loading time 0.00 +[Apr 20, 2026 - 10:57:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 452 : avg forward pass time 15.34 +[Apr 20, 2026 - 10:57:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 452 : backward pass time 12.39 +[Apr 20, 2026 - 10:57:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 452 : optimizer step time 1.56 +[Apr 20, 2026 - 10:57:49 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 452-------------------- +[Apr 20, 2026 - 10:57:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0101 iter count: 1.0 +[Apr 20, 2026 - 10:57:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[Apr 20, 2026 - 10:57:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 10:57:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 10:58:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.122694 43.231163 42.992485 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.558594 72.654297 69.607422 +util 96.250000 99.000000 94.000000 +[Apr 20, 2026 - 10:58:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 453 : avg iteration time 55.01 seconds +[Apr 20, 2026 - 10:58:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 453 : data loading time 0.00 +[Apr 20, 2026 - 10:58:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 453 : avg forward pass time 15.46 +[Apr 20, 2026 - 10:58:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 453 : backward pass time 11.56 +[Apr 20, 2026 - 10:58:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 453 : optimizer step time 0.98 +[Apr 20, 2026 - 10:58:44 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 453-------------------- +[Apr 20, 2026 - 10:58:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0095 iter count: 1.0 +[Apr 20, 2026 - 10:58:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[Apr 20, 2026 - 10:58:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 10:58:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 10:59:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.329264 43.444645 43.197914 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.534180 72.673828 69.470703 +util 97.750000 99.000000 95.000000 +[Apr 20, 2026 - 10:59:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 454 : avg iteration time 65.66 seconds +[Apr 20, 2026 - 10:59:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 454 : data loading time 0.00 +[Apr 20, 2026 - 10:59:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 454 : avg forward pass time 19.46 +[Apr 20, 2026 - 10:59:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 454 : backward pass time 12.74 +[Apr 20, 2026 - 10:59:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 454 : optimizer step time 1.04 +[Apr 20, 2026 - 10:59:50 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 454-------------------- +[Apr 20, 2026 - 10:59:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0118 iter count: 1.0 +[Apr 20, 2026 - 10:59:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0118 iter count: 1.0 +[Apr 20, 2026 - 10:59:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 10:59:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 11:03:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.329523 43.443871 43.198631 +peak_gpu_mem_gb 129.547819 131.549380 123.547254 +peak_gpu_mem_reserved_gb 133.550293 135.535156 127.746094 +util 71.750000 75.000000 68.000000 +[Apr 20, 2026 - 11:03:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 455 : avg iteration time 192.22 seconds +[Apr 20, 2026 - 11:03:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 455 : data loading time 0.00 +[Apr 20, 2026 - 11:03:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 455 : avg forward pass time 86.88 +[Apr 20, 2026 - 11:03:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 455 : backward pass time 9.80 +[Apr 20, 2026 - 11:03:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 455 : optimizer step time 0.34 +[Apr 20, 2026 - 11:03:02 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 455-------------------- +[Apr 20, 2026 - 11:03:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 211717.2402 iter count: 1.0 +[Apr 20, 2026 - 11:03:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6658 iter count: 1.0 +[Apr 20, 2026 - 11:03:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 211717.2402iter count: 1.0 +[Apr 20, 2026 - 11:03:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 11:03:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.3487iter count: 1.0 +[Apr 20, 2026 - 11:03:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.5875 iter count: 1.0 +[Apr 20, 2026 - 11:03:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.7401 iter count: 1.0 +[Apr 20, 2026 - 11:03:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 255647.0781iter count: 1.0 +[Apr 20, 2026 - 11:03:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 3.2379 iter count: 1.0 +[Apr 20, 2026 - 11:03:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 5284515.0000iter count: 1.0 +[Apr 20, 2026 - 11:03:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6658iter count: 1.0 +[Apr 20, 2026 - 11:03:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 5.9940 iter count: 1.0 +[Apr 20, 2026 - 11:03:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 160810.7812iter count: 1.0 +[Apr 20, 2026 - 11:03:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 2.7975 iter count: 1.0 +[Apr 20, 2026 - 11:03:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 269.0698iter count: 1.0 +[Apr 20, 2026 - 11:03:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.2811 iter count: 1.0 +[Apr 20, 2026 - 11:03:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 5230.2480iter count: 1.0 +[Apr 20, 2026 - 11:03:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 0.5875 iter count: 1.0 +[Apr 20, 2026 - 11:03:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 3.2379 iter count: 1.0 +[Apr 20, 2026 - 11:03:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[Apr 20, 2026 - 11:03:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 11:03:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.330175 43.444698 43.198936 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.455566 72.085938 69.802734 +util 94.250000 98.000000 90.000000 +[Apr 20, 2026 - 11:03:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 456 : avg iteration time 56.92 seconds +[Apr 20, 2026 - 11:03:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 456 : data loading time 0.00 +[Apr 20, 2026 - 11:03:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 456 : avg forward pass time 16.36 +[Apr 20, 2026 - 11:03:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 456 : backward pass time 11.55 +[Apr 20, 2026 - 11:03:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 456 : optimizer step time 1.00 +[Apr 20, 2026 - 11:03:59 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 456-------------------- +[Apr 20, 2026 - 11:03:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0124 iter count: 1.0 +[Apr 20, 2026 - 11:03:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0124 iter count: 1.0 +[Apr 20, 2026 - 11:03:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 11:03:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 11:04:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.330863 43.446365 43.199505 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.396973 72.281250 69.451172 +util 96.500000 100.000000 91.000000 +[Apr 20, 2026 - 11:04:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 457 : avg iteration time 59.75 seconds +[Apr 20, 2026 - 11:04:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 457 : data loading time 0.00 +[Apr 20, 2026 - 11:04:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 457 : avg forward pass time 16.96 +[Apr 20, 2026 - 11:04:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 457 : backward pass time 12.86 +[Apr 20, 2026 - 11:04:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 457 : optimizer step time 1.40 +[Apr 20, 2026 - 11:04:59 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 457-------------------- +[Apr 20, 2026 - 11:04:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0104 iter count: 1.0 +[Apr 20, 2026 - 11:04:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[Apr 20, 2026 - 11:04:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 11:04:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 11:06:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.331404 43.447327 43.200592 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.815918 70.035156 69.607422 +util 98.500000 100.000000 96.000000 +[Apr 20, 2026 - 11:06:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 458 : avg iteration time 63.31 seconds +[Apr 20, 2026 - 11:06:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 458 : data loading time 0.00 +[Apr 20, 2026 - 11:06:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 458 : avg forward pass time 19.59 +[Apr 20, 2026 - 11:06:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 458 : backward pass time 11.57 +[Apr 20, 2026 - 11:06:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 458 : optimizer step time 0.99 +[Apr 20, 2026 - 11:06:02 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 458-------------------- +[Apr 20, 2026 - 11:06:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0117 iter count: 1.0 +[Apr 20, 2026 - 11:06:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[Apr 20, 2026 - 11:06:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 11:06:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 11:06:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.331486 43.447334 43.200726 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.767090 70.035156 69.509766 +util 94.750000 99.000000 91.000000 +[Apr 20, 2026 - 11:06:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 459 : avg iteration time 56.54 seconds +[Apr 20, 2026 - 11:06:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 459 : data loading time 0.00 +[Apr 20, 2026 - 11:06:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 459 : avg forward pass time 15.87 +[Apr 20, 2026 - 11:06:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 459 : backward pass time 12.23 +[Apr 20, 2026 - 11:06:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 459 : optimizer step time 1.01 +[Apr 20, 2026 - 11:06:59 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 459-------------------- +[Apr 20, 2026 - 11:06:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0116 iter count: 1.0 +[Apr 20, 2026 - 11:06:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[Apr 20, 2026 - 11:06:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 11:06:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 11:10:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.331723 43.447136 43.200897 +peak_gpu_mem_gb 129.547819 131.549380 123.547254 +peak_gpu_mem_reserved_gb 133.535645 135.576172 127.548828 +util 69.500000 73.000000 66.000000 +[Apr 20, 2026 - 11:10:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 460 : avg iteration time 186.45 seconds +[Apr 20, 2026 - 11:10:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 460 : data loading time 0.00 +[Apr 20, 2026 - 11:10:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 460 : avg forward pass time 84.62 +[Apr 20, 2026 - 11:10:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 460 : backward pass time 8.38 +[Apr 20, 2026 - 11:10:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 460 : optimizer step time 0.34 +[Apr 20, 2026 - 11:10:05 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 460-------------------- +[Apr 20, 2026 - 11:10:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 228205.4971 iter count: 1.0 +[Apr 20, 2026 - 11:10:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.7091 iter count: 1.0 +[Apr 20, 2026 - 11:10:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 228205.4971iter count: 1.0 +[Apr 20, 2026 - 11:10:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 11:10:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.6511iter count: 1.0 +[Apr 20, 2026 - 11:10:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.5640 iter count: 1.0 +[Apr 20, 2026 - 11:10:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.6701 iter count: 1.0 +[Apr 20, 2026 - 11:10:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 271755.7188iter count: 1.0 +[Apr 20, 2026 - 11:10:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 3.0892 iter count: 1.0 +[Apr 20, 2026 - 11:10:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 4593968.5000iter count: 1.0 +[Apr 20, 2026 - 11:10:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.7091iter count: 1.0 +[Apr 20, 2026 - 11:10:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 6.8392 iter count: 1.0 +[Apr 20, 2026 - 11:10:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 871905.4375iter count: 1.0 +[Apr 20, 2026 - 11:10:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 5.3596 iter count: 1.0 +[Apr 20, 2026 - 11:10:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 45212.1406iter count: 1.0 +[Apr 20, 2026 - 11:10:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.5079 iter count: 1.0 +[Apr 20, 2026 - 11:10:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 8231.5684iter count: 1.0 +[Apr 20, 2026 - 11:10:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 7.1279 iter count: 1.0 +[Apr 20, 2026 - 11:10:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 1553296.3750iter count: 1.0 +[Apr 20, 2026 - 11:10:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0123 iter count: 1.0 +[Apr 20, 2026 - 11:10:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 11:10:54 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.332341 43.447269 43.201851 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.845215 70.113281 69.607422 +util 94.750000 99.000000 91.000000 +[Apr 20, 2026 - 11:10:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 461 : avg iteration time 49.15 seconds +[Apr 20, 2026 - 11:10:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 461 : data loading time 0.00 +[Apr 20, 2026 - 11:10:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 461 : avg forward pass time 11.94 +[Apr 20, 2026 - 11:10:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 461 : backward pass time 11.59 +[Apr 20, 2026 - 11:10:54 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 461 : optimizer step time 0.98 +[Apr 20, 2026 - 11:10:54 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 461-------------------- +[Apr 20, 2026 - 11:10:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0127 iter count: 1.0 +[Apr 20, 2026 - 11:10:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0127 iter count: 1.0 +[Apr 20, 2026 - 11:10:54 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 11:10:54 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 11:11:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.332767 43.447502 43.201912 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.426270 72.361328 69.703125 +util 96.500000 100.000000 91.000000 +[Apr 20, 2026 - 11:11:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 462 : avg iteration time 58.95 seconds +[Apr 20, 2026 - 11:11:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 462 : data loading time 0.00 +[Apr 20, 2026 - 11:11:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 462 : avg forward pass time 17.29 +[Apr 20, 2026 - 11:11:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 462 : backward pass time 11.77 +[Apr 20, 2026 - 11:11:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 462 : optimizer step time 0.99 +[Apr 20, 2026 - 11:11:53 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 462-------------------- +[Apr 20, 2026 - 11:11:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0106 iter count: 1.0 +[Apr 20, 2026 - 11:11:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[Apr 20, 2026 - 11:11:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 11:11:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 11:12:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.333414 43.449219 43.201904 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.708496 69.917969 69.431641 +util 95.500000 99.000000 92.000000 +[Apr 20, 2026 - 11:12:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 463 : avg iteration time 52.85 seconds +[Apr 20, 2026 - 11:12:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 463 : data loading time 0.00 +[Apr 20, 2026 - 11:12:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 463 : avg forward pass time 14.15 +[Apr 20, 2026 - 11:12:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 463 : backward pass time 11.57 +[Apr 20, 2026 - 11:12:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 463 : optimizer step time 0.98 +[Apr 20, 2026 - 11:12:46 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 463-------------------- +[Apr 20, 2026 - 11:12:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0123 iter count: 1.0 +[Apr 20, 2026 - 11:12:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0123 iter count: 1.0 +[Apr 20, 2026 - 11:12:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 11:12:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 11:13:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.333701 43.450130 43.202129 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.431152 72.322266 69.722656 +util 99.250000 100.000000 97.000000 +[Apr 20, 2026 - 11:13:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 464 : avg iteration time 54.06 seconds +[Apr 20, 2026 - 11:13:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 464 : data loading time 0.00 +[Apr 20, 2026 - 11:13:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 464 : avg forward pass time 14.24 +[Apr 20, 2026 - 11:13:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 464 : backward pass time 12.49 +[Apr 20, 2026 - 11:13:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 464 : optimizer step time 1.53 +[Apr 20, 2026 - 11:13:40 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 464-------------------- +[Apr 20, 2026 - 11:13:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0121 iter count: 1.0 +[Apr 20, 2026 - 11:13:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[Apr 20, 2026 - 11:13:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 11:13:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 11:16:55 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.333491 43.448967 43.202133 +peak_gpu_mem_gb 129.547819 131.549380 123.547254 +peak_gpu_mem_reserved_gb 133.975098 135.593750 129.269531 +util 63.000000 69.000000 55.000000 +[Apr 20, 2026 - 11:16:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 465 : avg iteration time 194.64 seconds +[Apr 20, 2026 - 11:16:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 465 : data loading time 0.00 +[Apr 20, 2026 - 11:16:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 465 : avg forward pass time 88.02 +[Apr 20, 2026 - 11:16:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 465 : backward pass time 9.94 +[Apr 20, 2026 - 11:16:55 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 465 : optimizer step time 0.36 +[Apr 20, 2026 - 11:16:55 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 465-------------------- +[Apr 20, 2026 - 11:16:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 27782.5557 iter count: 1.0 +[Apr 20, 2026 - 11:16:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6713 iter count: 1.0 +[Apr 20, 2026 - 11:16:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 27782.5557iter count: 1.0 +[Apr 20, 2026 - 11:16:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 11:16:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.1946iter count: 1.0 +[Apr 20, 2026 - 11:16:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.2727 iter count: 1.0 +[Apr 20, 2026 - 11:16:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.3170 iter count: 1.0 +[Apr 20, 2026 - 11:16:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 38165.0508iter count: 1.0 +[Apr 20, 2026 - 11:16:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 12.7482iter count: 1.0 +[Apr 20, 2026 - 11:16:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 306791.7500iter count: 1.0 +[Apr 20, 2026 - 11:16:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6713iter count: 1.0 +[Apr 20, 2026 - 11:16:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 1.5502 iter count: 1.0 +[Apr 20, 2026 - 11:16:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 22.2080 iter count: 1.0 +[Apr 20, 2026 - 11:16:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.4720 iter count: 1.0 +[Apr 20, 2026 - 11:16:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 1036.9988iter count: 1.0 +[Apr 20, 2026 - 11:16:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.2859 iter count: 1.0 +[Apr 20, 2026 - 11:16:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 5280.3257iter count: 1.0 +[Apr 20, 2026 - 11:16:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.7185 iter count: 1.0 +[Apr 20, 2026 - 11:16:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 12543.3857iter count: 1.0 +[Apr 20, 2026 - 11:16:55 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0148 iter count: 1.0 +[Apr 20, 2026 - 11:16:55 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 11:17:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.333774 43.449520 43.202366 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.606934 72.632812 69.861328 +util 95.500000 99.000000 92.000000 +[Apr 20, 2026 - 11:17:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 466 : avg iteration time 51.10 seconds +[Apr 20, 2026 - 11:17:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 466 : data loading time 0.00 +[Apr 20, 2026 - 11:17:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 466 : avg forward pass time 13.52 +[Apr 20, 2026 - 11:17:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 466 : backward pass time 11.53 +[Apr 20, 2026 - 11:17:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 466 : optimizer step time 0.99 +[Apr 20, 2026 - 11:17:46 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 466-------------------- +[Apr 20, 2026 - 11:17:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0137 iter count: 1.0 +[Apr 20, 2026 - 11:17:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0137 iter count: 1.0 +[Apr 20, 2026 - 11:17:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 11:17:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 11:18:45 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.334196 43.450794 43.202568 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.665527 72.671875 69.763672 +util 97.250000 100.000000 91.000000 +[Apr 20, 2026 - 11:18:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 467 : avg iteration time 58.71 seconds +[Apr 20, 2026 - 11:18:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 467 : data loading time 0.00 +[Apr 20, 2026 - 11:18:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 467 : avg forward pass time 16.55 +[Apr 20, 2026 - 11:18:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 467 : backward pass time 11.56 +[Apr 20, 2026 - 11:18:45 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 467 : optimizer step time 1.24 +[Apr 20, 2026 - 11:18:45 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 467-------------------- +[Apr 20, 2026 - 11:18:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0126 iter count: 1.0 +[Apr 20, 2026 - 11:18:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0126 iter count: 1.0 +[Apr 20, 2026 - 11:18:45 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 11:18:45 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 11:19:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.334389 43.451084 43.202892 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.675293 72.671875 69.822266 +util 99.250000 100.000000 99.000000 +[Apr 20, 2026 - 11:19:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 468 : avg iteration time 55.57 seconds +[Apr 20, 2026 - 11:19:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 468 : data loading time 0.00 +[Apr 20, 2026 - 11:19:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 468 : avg forward pass time 15.71 +[Apr 20, 2026 - 11:19:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 468 : backward pass time 11.55 +[Apr 20, 2026 - 11:19:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 468 : optimizer step time 0.99 +[Apr 20, 2026 - 11:19:40 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 468-------------------- +[Apr 20, 2026 - 11:19:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0122 iter count: 1.0 +[Apr 20, 2026 - 11:19:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0122 iter count: 1.0 +[Apr 20, 2026 - 11:19:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 11:19:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 11:20:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.334534 43.451336 43.203110 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.660645 72.671875 69.783203 +util 97.500000 99.000000 93.000000 +[Apr 20, 2026 - 11:20:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 469 : avg iteration time 56.25 seconds +[Apr 20, 2026 - 11:20:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 469 : data loading time 0.00 +[Apr 20, 2026 - 11:20:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 469 : avg forward pass time 15.61 +[Apr 20, 2026 - 11:20:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 469 : backward pass time 11.51 +[Apr 20, 2026 - 11:20:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 469 : optimizer step time 1.02 +[Apr 20, 2026 - 11:20:36 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 469-------------------- +[Apr 20, 2026 - 11:20:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0134 iter count: 1.0 +[Apr 20, 2026 - 11:20:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0134 iter count: 1.0 +[Apr 20, 2026 - 11:20:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 11:20:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 11:23:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.334963 43.450134 43.204891 +peak_gpu_mem_gb 129.547819 131.549380 123.547254 +peak_gpu_mem_reserved_gb 133.603516 135.613281 127.863281 +util 67.000000 71.000000 64.000000 +[Apr 20, 2026 - 11:23:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 470 : avg iteration time 196.69 seconds +[Apr 20, 2026 - 11:23:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 470 : data loading time 0.00 +[Apr 20, 2026 - 11:23:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 470 : avg forward pass time 89.80 +[Apr 20, 2026 - 11:23:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 470 : backward pass time 8.48 +[Apr 20, 2026 - 11:23:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 470 : optimizer step time 0.34 +[Apr 20, 2026 - 11:23:53 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 470-------------------- +[Apr 20, 2026 - 11:23:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 24690.1555 iter count: 1.0 +[Apr 20, 2026 - 11:23:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6420 iter count: 1.0 +[Apr 20, 2026 - 11:23:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 24690.1555iter count: 1.0 +[Apr 20, 2026 - 11:23:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 11:23:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.9998iter count: 1.0 +[Apr 20, 2026 - 11:23:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.0143 iter count: 1.0 +[Apr 20, 2026 - 11:23:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.5048 iter count: 1.0 +[Apr 20, 2026 - 11:23:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 47138.4219iter count: 1.0 +[Apr 20, 2026 - 11:23:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 7.6035 iter count: 1.0 +[Apr 20, 2026 - 11:23:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 446688.5312iter count: 1.0 +[Apr 20, 2026 - 11:23:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6420iter count: 1.0 +[Apr 20, 2026 - 11:23:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 6.0352 iter count: 1.0 +[Apr 20, 2026 - 11:23:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 174625.7344iter count: 1.0 +[Apr 20, 2026 - 11:23:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.2631 iter count: 1.0 +[Apr 20, 2026 - 11:23:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 682.7943iter count: 1.0 +[Apr 20, 2026 - 11:23:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 5.3274 iter count: 1.0 +[Apr 20, 2026 - 11:23:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 42395.7852iter count: 1.0 +[Apr 20, 2026 - 11:23:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 1.4101 iter count: 1.0 +[Apr 20, 2026 - 11:23:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 16.7809 iter count: 1.0 +[Apr 20, 2026 - 11:23:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0142 iter count: 1.0 +[Apr 20, 2026 - 11:23:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 11:24:50 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.335199 43.450386 43.204971 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.854980 69.957031 69.705078 +util 99.500000 100.000000 99.000000 +[Apr 20, 2026 - 11:24:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 471 : avg iteration time 56.66 seconds +[Apr 20, 2026 - 11:24:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 471 : data loading time 0.00 +[Apr 20, 2026 - 11:24:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 471 : avg forward pass time 16.29 +[Apr 20, 2026 - 11:24:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 471 : backward pass time 11.53 +[Apr 20, 2026 - 11:24:50 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 471 : optimizer step time 0.99 +[Apr 20, 2026 - 11:24:50 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 471-------------------- +[Apr 20, 2026 - 11:24:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0119 iter count: 1.0 +[Apr 20, 2026 - 11:24:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[Apr 20, 2026 - 11:24:50 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 11:24:50 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 11:25:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.335781 43.450581 43.205017 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.903809 69.976562 69.800781 +util 98.250000 100.000000 95.000000 +[Apr 20, 2026 - 11:25:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 472 : avg iteration time 53.36 seconds +[Apr 20, 2026 - 11:25:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 472 : data loading time 0.00 +[Apr 20, 2026 - 11:25:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 472 : avg forward pass time 14.01 +[Apr 20, 2026 - 11:25:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 472 : backward pass time 11.59 +[Apr 20, 2026 - 11:25:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 472 : optimizer step time 1.25 +[Apr 20, 2026 - 11:25:43 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 472-------------------- +[Apr 20, 2026 - 11:25:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0110 iter count: 1.0 +[Apr 20, 2026 - 11:25:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[Apr 20, 2026 - 11:25:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 11:25:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 11:26:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.335857 43.450829 43.205063 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.864746 69.976562 69.685547 +util 99.500000 100.000000 99.000000 +[Apr 20, 2026 - 11:26:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 473 : avg iteration time 52.84 seconds +[Apr 20, 2026 - 11:26:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 473 : data loading time 0.00 +[Apr 20, 2026 - 11:26:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 473 : avg forward pass time 14.35 +[Apr 20, 2026 - 11:26:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 473 : backward pass time 11.58 +[Apr 20, 2026 - 11:26:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 473 : optimizer step time 0.99 +[Apr 20, 2026 - 11:26:36 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 473-------------------- +[Apr 20, 2026 - 11:26:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0118 iter count: 1.0 +[Apr 20, 2026 - 11:26:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0118 iter count: 1.0 +[Apr 20, 2026 - 11:26:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 11:26:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 11:27:35 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.335964 43.450951 43.205177 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.869629 69.976562 69.724609 +util 97.500000 100.000000 92.000000 +[Apr 20, 2026 - 11:27:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 474 : avg iteration time 58.65 seconds +[Apr 20, 2026 - 11:27:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 474 : data loading time 0.00 +[Apr 20, 2026 - 11:27:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 474 : avg forward pass time 16.86 +[Apr 20, 2026 - 11:27:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 474 : backward pass time 11.54 +[Apr 20, 2026 - 11:27:35 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 474 : optimizer step time 1.02 +[Apr 20, 2026 - 11:27:35 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 474-------------------- +[Apr 20, 2026 - 11:27:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0111 iter count: 1.0 +[Apr 20, 2026 - 11:27:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[Apr 20, 2026 - 11:27:35 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 11:27:35 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 11:30:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.335753 43.450111 43.205173 +peak_gpu_mem_gb 129.547819 131.549380 123.547254 +peak_gpu_mem_reserved_gb 133.608887 135.593750 127.882812 +util 67.750000 70.000000 65.000000 +[Apr 20, 2026 - 11:30:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 475 : avg iteration time 196.90 seconds +[Apr 20, 2026 - 11:30:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 475 : data loading time 0.00 +[Apr 20, 2026 - 11:30:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 475 : avg forward pass time 89.92 +[Apr 20, 2026 - 11:30:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 475 : backward pass time 8.42 +[Apr 20, 2026 - 11:30:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 475 : optimizer step time 0.34 +[Apr 20, 2026 - 11:30:52 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 475-------------------- +[Apr 20, 2026 - 11:30:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 111696.6665 iter count: 1.0 +[Apr 20, 2026 - 11:30:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6431 iter count: 1.0 +[Apr 20, 2026 - 11:30:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 111696.6665iter count: 1.0 +[Apr 20, 2026 - 11:30:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 11:30:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.7537iter count: 1.0 +[Apr 20, 2026 - 11:30:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.0152 iter count: 1.0 +[Apr 20, 2026 - 11:30:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.4491 iter count: 1.0 +[Apr 20, 2026 - 11:30:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 138926.0938iter count: 1.0 +[Apr 20, 2026 - 11:30:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 7.6165 iter count: 1.0 +[Apr 20, 2026 - 11:30:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 2952320.0000iter count: 1.0 +[Apr 20, 2026 - 11:30:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6431iter count: 1.0 +[Apr 20, 2026 - 11:30:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 6.4530 iter count: 1.0 +[Apr 20, 2026 - 11:30:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 402748.2188iter count: 1.0 +[Apr 20, 2026 - 11:30:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 2.9985 iter count: 1.0 +[Apr 20, 2026 - 11:30:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 402.1980iter count: 1.0 +[Apr 20, 2026 - 11:30:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 2.8737 iter count: 1.0 +[Apr 20, 2026 - 11:30:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 313.3522iter count: 1.0 +[Apr 20, 2026 - 11:30:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 2.5829 iter count: 1.0 +[Apr 20, 2026 - 11:30:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 175.1789iter count: 1.0 +[Apr 20, 2026 - 11:30:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0123 iter count: 1.0 +[Apr 20, 2026 - 11:30:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.02 seconds +[Apr 20, 2026 - 11:31:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.335827 43.450138 43.205349 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.616699 72.496094 69.841797 +util 94.750000 99.000000 91.000000 +[Apr 20, 2026 - 11:31:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 476 : avg iteration time 59.96 seconds +[Apr 20, 2026 - 11:31:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 476 : data loading time 0.01 +[Apr 20, 2026 - 11:31:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 476 : avg forward pass time 17.93 +[Apr 20, 2026 - 11:31:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 476 : backward pass time 11.55 +[Apr 20, 2026 - 11:31:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 476 : optimizer step time 0.97 +[Apr 20, 2026 - 11:31:52 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 476-------------------- +[Apr 20, 2026 - 11:31:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0108 iter count: 1.0 +[Apr 20, 2026 - 11:31:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[Apr 20, 2026 - 11:31:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 11:31:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 11:32:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.336289 43.451878 43.205322 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.631348 72.652344 69.822266 +util 98.500000 100.000000 95.000000 +[Apr 20, 2026 - 11:32:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 477 : avg iteration time 56.24 seconds +[Apr 20, 2026 - 11:32:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 477 : data loading time 0.00 +[Apr 20, 2026 - 11:32:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 477 : avg forward pass time 15.16 +[Apr 20, 2026 - 11:32:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 477 : backward pass time 11.67 +[Apr 20, 2026 - 11:32:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 477 : optimizer step time 1.41 +[Apr 20, 2026 - 11:32:48 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 477-------------------- +[Apr 20, 2026 - 11:32:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0121 iter count: 1.0 +[Apr 20, 2026 - 11:32:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[Apr 20, 2026 - 11:32:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 11:32:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 11:33:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.336400 43.452274 43.205345 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.889160 70.210938 69.664062 +util 96.000000 99.000000 91.000000 +[Apr 20, 2026 - 11:33:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 478 : avg iteration time 55.72 seconds +[Apr 20, 2026 - 11:33:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 478 : data loading time 0.00 +[Apr 20, 2026 - 11:33:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 478 : avg forward pass time 15.80 +[Apr 20, 2026 - 11:33:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 478 : backward pass time 11.55 +[Apr 20, 2026 - 11:33:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 478 : optimizer step time 1.00 +[Apr 20, 2026 - 11:33:43 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 478-------------------- +[Apr 20, 2026 - 11:33:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0104 iter count: 1.0 +[Apr 20, 2026 - 11:33:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[Apr 20, 2026 - 11:33:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 11:33:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 11:34:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.336386 43.452087 43.205406 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.025879 70.210938 69.880859 +util 99.500000 100.000000 99.000000 +[Apr 20, 2026 - 11:34:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 479 : avg iteration time 48.13 seconds +[Apr 20, 2026 - 11:34:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 479 : data loading time 0.00 +[Apr 20, 2026 - 11:34:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 479 : avg forward pass time 11.48 +[Apr 20, 2026 - 11:34:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 479 : backward pass time 11.56 +[Apr 20, 2026 - 11:34:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 479 : optimizer step time 0.99 +[Apr 20, 2026 - 11:34:32 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 479-------------------- +[Apr 20, 2026 - 11:34:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0114 iter count: 1.0 +[Apr 20, 2026 - 11:34:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[Apr 20, 2026 - 11:34:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 11:34:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 11:37:46 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.336370 43.451790 43.205482 +peak_gpu_mem_gb 129.547819 131.549380 123.547254 +peak_gpu_mem_reserved_gb 133.833496 135.554688 128.839844 +util 71.500000 75.000000 68.000000 +[Apr 20, 2026 - 11:37:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 480 : avg iteration time 194.19 seconds +[Apr 20, 2026 - 11:37:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 480 : data loading time 0.00 +[Apr 20, 2026 - 11:37:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 480 : avg forward pass time 88.57 +[Apr 20, 2026 - 11:37:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 480 : backward pass time 8.33 +[Apr 20, 2026 - 11:37:46 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 480 : optimizer step time 0.34 +[Apr 20, 2026 - 11:37:46 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 480-------------------- +[Apr 20, 2026 - 11:37:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 210250.9683 iter count: 1.0 +[Apr 20, 2026 - 11:37:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6145 iter count: 1.0 +[Apr 20, 2026 - 11:37:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 210250.9683iter count: 1.0 +[Apr 20, 2026 - 11:37:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 11:37:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.3991iter count: 1.0 +[Apr 20, 2026 - 11:37:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.5328 iter count: 1.0 +[Apr 20, 2026 - 11:37:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.7991 iter count: 1.0 +[Apr 20, 2026 - 11:37:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 335370.3125iter count: 1.0 +[Apr 20, 2026 - 11:37:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 2.9023 iter count: 1.0 +[Apr 20, 2026 - 11:37:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 5945813.5000iter count: 1.0 +[Apr 20, 2026 - 11:37:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6145iter count: 1.0 +[Apr 20, 2026 - 11:37:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.7668 iter count: 1.0 +[Apr 20, 2026 - 11:37:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 1869.9601iter count: 1.0 +[Apr 20, 2026 - 11:37:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 5.1640 iter count: 1.0 +[Apr 20, 2026 - 11:37:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 30579.4355iter count: 1.0 +[Apr 20, 2026 - 11:37:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 3.3018 iter count: 1.0 +[Apr 20, 2026 - 11:37:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 737.7629iter count: 1.0 +[Apr 20, 2026 - 11:37:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 6.4825 iter count: 1.0 +[Apr 20, 2026 - 11:37:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 427218.8438iter count: 1.0 +[Apr 20, 2026 - 11:37:46 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[Apr 20, 2026 - 11:37:46 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 11:38:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.336414 43.451904 43.205498 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.986816 70.054688 69.917969 +util 98.000000 100.000000 92.000000 +[Apr 20, 2026 - 11:38:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 481 : avg iteration time 52.85 seconds +[Apr 20, 2026 - 11:38:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 481 : data loading time 0.00 +[Apr 20, 2026 - 11:38:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 481 : avg forward pass time 13.74 +[Apr 20, 2026 - 11:38:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 481 : backward pass time 12.18 +[Apr 20, 2026 - 11:38:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 481 : optimizer step time 1.56 +[Apr 20, 2026 - 11:38:39 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 481-------------------- +[Apr 20, 2026 - 11:38:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0092 iter count: 1.0 +[Apr 20, 2026 - 11:38:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0092 iter count: 1.0 +[Apr 20, 2026 - 11:38:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 11:38:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 11:39:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.336522 43.452126 43.205631 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.864746 70.054688 69.605469 +util 99.000000 99.000000 99.000000 +[Apr 20, 2026 - 11:39:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 482 : avg iteration time 50.16 seconds +[Apr 20, 2026 - 11:39:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 482 : data loading time 0.00 +[Apr 20, 2026 - 11:39:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 482 : avg forward pass time 12.66 +[Apr 20, 2026 - 11:39:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 482 : backward pass time 12.29 +[Apr 20, 2026 - 11:39:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 482 : optimizer step time 0.99 +[Apr 20, 2026 - 11:39:29 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 482-------------------- +[Apr 20, 2026 - 11:39:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0099 iter count: 1.0 +[Apr 20, 2026 - 11:39:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[Apr 20, 2026 - 11:39:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 11:39:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 11:40:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.336601 43.452141 43.205631 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.835449 69.937500 69.605469 +util 97.500000 100.000000 92.000000 +[Apr 20, 2026 - 11:40:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 483 : avg iteration time 56.85 seconds +[Apr 20, 2026 - 11:40:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 483 : data loading time 0.00 +[Apr 20, 2026 - 11:40:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 483 : avg forward pass time 15.89 +[Apr 20, 2026 - 11:40:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 483 : backward pass time 11.62 +[Apr 20, 2026 - 11:40:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 483 : optimizer step time 1.00 +[Apr 20, 2026 - 11:40:26 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 483-------------------- +[Apr 20, 2026 - 11:40:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0120 iter count: 1.0 +[Apr 20, 2026 - 11:40:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0120 iter count: 1.0 +[Apr 20, 2026 - 11:40:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 11:40:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 11:41:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.336681 43.452152 43.205769 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.762207 69.839844 69.605469 +util 96.000000 99.000000 93.000000 +[Apr 20, 2026 - 11:41:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 484 : avg iteration time 47.88 seconds +[Apr 20, 2026 - 11:41:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 484 : data loading time 0.00 +[Apr 20, 2026 - 11:41:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 484 : avg forward pass time 11.52 +[Apr 20, 2026 - 11:41:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 484 : backward pass time 11.63 +[Apr 20, 2026 - 11:41:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 484 : optimizer step time 0.98 +[Apr 20, 2026 - 11:41:14 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 484-------------------- +[Apr 20, 2026 - 11:41:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0111 iter count: 1.0 +[Apr 20, 2026 - 11:41:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[Apr 20, 2026 - 11:41:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 11:41:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 11:44:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.336344 43.450794 43.205753 +peak_gpu_mem_gb 129.547819 131.549380 123.547254 +peak_gpu_mem_reserved_gb 133.584961 135.576172 127.726562 +util 66.500000 78.000000 49.000000 +[Apr 20, 2026 - 11:44:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 485 : avg iteration time 193.91 seconds +[Apr 20, 2026 - 11:44:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 485 : data loading time 0.00 +[Apr 20, 2026 - 11:44:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 485 : avg forward pass time 88.17 +[Apr 20, 2026 - 11:44:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 485 : backward pass time 8.93 +[Apr 20, 2026 - 11:44:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 485 : optimizer step time 0.34 +[Apr 20, 2026 - 11:44:27 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 485-------------------- +[Apr 20, 2026 - 11:44:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 52408.4309 iter count: 1.0 +[Apr 20, 2026 - 11:44:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5907 iter count: 1.0 +[Apr 20, 2026 - 11:44:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 52408.4309iter count: 1.0 +[Apr 20, 2026 - 11:44:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 11:44:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.8935iter count: 1.0 +[Apr 20, 2026 - 11:44:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.9262 iter count: 1.0 +[Apr 20, 2026 - 11:44:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.2713 iter count: 1.0 +[Apr 20, 2026 - 11:44:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 88338.7734iter count: 1.0 +[Apr 20, 2026 - 11:44:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 6.3745 iter count: 1.0 +[Apr 20, 2026 - 11:44:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 2069047.1250iter count: 1.0 +[Apr 20, 2026 - 11:44:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5907iter count: 1.0 +[Apr 20, 2026 - 11:44:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 1.4514 iter count: 1.0 +[Apr 20, 2026 - 11:44:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 18.2264 iter count: 1.0 +[Apr 20, 2026 - 11:44:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 5.1342 iter count: 1.0 +[Apr 20, 2026 - 11:44:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 28810.5645iter count: 1.0 +[Apr 20, 2026 - 11:44:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 5.2471 iter count: 1.0 +[Apr 20, 2026 - 11:44:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 36103.6836iter count: 1.0 +[Apr 20, 2026 - 11:44:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 6.2973 iter count: 1.0 +[Apr 20, 2026 - 11:44:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 294971.2188iter count: 1.0 +[Apr 20, 2026 - 11:44:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[Apr 20, 2026 - 11:44:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 11:45:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.336344 43.450783 43.205753 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.854980 69.976562 69.742188 +util 97.500000 100.000000 92.000000 +[Apr 20, 2026 - 11:45:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 486 : avg iteration time 56.99 seconds +[Apr 20, 2026 - 11:45:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 486 : data loading time 0.00 +[Apr 20, 2026 - 11:45:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 486 : avg forward pass time 16.41 +[Apr 20, 2026 - 11:45:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 486 : backward pass time 11.62 +[Apr 20, 2026 - 11:45:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 486 : optimizer step time 0.98 +[Apr 20, 2026 - 11:45:24 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 486-------------------- +[Apr 20, 2026 - 11:45:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0101 iter count: 1.0 +[Apr 20, 2026 - 11:45:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[Apr 20, 2026 - 11:45:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 11:45:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 11:46:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.336727 43.452053 43.205776 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.815918 69.937500 69.722656 +util 98.250000 100.000000 95.000000 +[Apr 20, 2026 - 11:46:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 487 : avg iteration time 61.92 seconds +[Apr 20, 2026 - 11:46:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 487 : data loading time 0.00 +[Apr 20, 2026 - 11:46:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 487 : avg forward pass time 17.89 +[Apr 20, 2026 - 11:46:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 487 : backward pass time 11.91 +[Apr 20, 2026 - 11:46:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 487 : optimizer step time 1.53 +[Apr 20, 2026 - 11:46:26 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 487-------------------- +[Apr 20, 2026 - 11:46:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0126 iter count: 1.0 +[Apr 20, 2026 - 11:46:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0126 iter count: 1.0 +[Apr 20, 2026 - 11:46:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 11:46:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 11:47:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.336786 43.452175 43.205875 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.801270 69.957031 69.605469 +util 97.500000 99.000000 93.000000 +[Apr 20, 2026 - 11:47:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 488 : avg iteration time 57.49 seconds +[Apr 20, 2026 - 11:47:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 488 : data loading time 0.00 +[Apr 20, 2026 - 11:47:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 488 : avg forward pass time 16.68 +[Apr 20, 2026 - 11:47:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 488 : backward pass time 11.61 +[Apr 20, 2026 - 11:47:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 488 : optimizer step time 0.98 +[Apr 20, 2026 - 11:47:24 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 488-------------------- +[Apr 20, 2026 - 11:47:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0105 iter count: 1.0 +[Apr 20, 2026 - 11:47:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[Apr 20, 2026 - 11:47:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 11:47:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 11:48:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.336949 43.452793 43.205875 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.820801 69.996094 69.625000 +util 95.500000 100.000000 92.000000 +[Apr 20, 2026 - 11:48:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 489 : avg iteration time 56.29 seconds +[Apr 20, 2026 - 11:48:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 489 : data loading time 0.00 +[Apr 20, 2026 - 11:48:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 489 : avg forward pass time 15.18 +[Apr 20, 2026 - 11:48:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 489 : backward pass time 12.34 +[Apr 20, 2026 - 11:48:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 489 : optimizer step time 1.30 +[Apr 20, 2026 - 11:48:20 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 489-------------------- +[Apr 20, 2026 - 11:48:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0130 iter count: 1.0 +[Apr 20, 2026 - 11:48:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0130 iter count: 1.0 +[Apr 20, 2026 - 11:48:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 11:48:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 11:51:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.336756 43.451725 43.206051 +peak_gpu_mem_gb 129.547819 131.549380 123.547254 +peak_gpu_mem_reserved_gb 133.604492 135.615234 127.785156 +util 65.500000 75.000000 53.000000 +[Apr 20, 2026 - 11:51:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 490 : avg iteration time 182.49 seconds +[Apr 20, 2026 - 11:51:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 490 : data loading time 0.00 +[Apr 20, 2026 - 11:51:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 490 : avg forward pass time 82.48 +[Apr 20, 2026 - 11:51:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 490 : backward pass time 8.85 +[Apr 20, 2026 - 11:51:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 490 : optimizer step time 0.34 +[Apr 20, 2026 - 11:51:23 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 490-------------------- +[Apr 20, 2026 - 11:51:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 19800.8234 iter count: 1.0 +[Apr 20, 2026 - 11:51:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6385 iter count: 1.0 +[Apr 20, 2026 - 11:51:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 19800.8234iter count: 1.0 +[Apr 20, 2026 - 11:51:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 11:51:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.0143iter count: 1.0 +[Apr 20, 2026 - 11:51:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.4125 iter count: 1.0 +[Apr 20, 2026 - 11:51:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.3822 iter count: 1.0 +[Apr 20, 2026 - 11:51:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 37815.5938iter count: 1.0 +[Apr 20, 2026 - 11:51:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 16.8596iter count: 1.0 +[Apr 20, 2026 - 11:51:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 349557.1875iter count: 1.0 +[Apr 20, 2026 - 11:51:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6385iter count: 1.0 +[Apr 20, 2026 - 11:51:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 5.3423 iter count: 1.0 +[Apr 20, 2026 - 11:51:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 43679.6250iter count: 1.0 +[Apr 20, 2026 - 11:51:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 5.0811 iter count: 1.0 +[Apr 20, 2026 - 11:51:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 25906.3867iter count: 1.0 +[Apr 20, 2026 - 11:51:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.8097 iter count: 1.0 +[Apr 20, 2026 - 11:51:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 15054.0723iter count: 1.0 +[Apr 20, 2026 - 11:51:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.3507 iter count: 1.0 +[Apr 20, 2026 - 11:51:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 6010.9146iter count: 1.0 +[Apr 20, 2026 - 11:51:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[Apr 20, 2026 - 11:51:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 11:52:27 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.336761 43.451710 43.206032 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 71.148926 72.417969 69.841797 +util 99.000000 100.000000 98.000000 +[Apr 20, 2026 - 11:52:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 491 : avg iteration time 64.56 seconds +[Apr 20, 2026 - 11:52:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 491 : data loading time 0.00 +[Apr 20, 2026 - 11:52:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 491 : avg forward pass time 20.24 +[Apr 20, 2026 - 11:52:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 491 : backward pass time 11.56 +[Apr 20, 2026 - 11:52:27 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 491 : optimizer step time 0.98 +[Apr 20, 2026 - 11:52:27 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 491-------------------- +[Apr 20, 2026 - 11:52:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0108 iter count: 1.0 +[Apr 20, 2026 - 11:52:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[Apr 20, 2026 - 11:52:27 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 11:52:27 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 11:53:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.336824 43.451721 43.206223 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.445801 72.183594 69.761719 +util 96.750000 100.000000 92.000000 +[Apr 20, 2026 - 11:53:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 492 : avg iteration time 56.50 seconds +[Apr 20, 2026 - 11:53:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 492 : data loading time 0.00 +[Apr 20, 2026 - 11:53:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 492 : avg forward pass time 15.66 +[Apr 20, 2026 - 11:53:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 492 : backward pass time 12.50 +[Apr 20, 2026 - 11:53:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 492 : optimizer step time 1.01 +[Apr 20, 2026 - 11:53:24 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 492-------------------- +[Apr 20, 2026 - 11:53:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0113 iter count: 1.0 +[Apr 20, 2026 - 11:53:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0113 iter count: 1.0 +[Apr 20, 2026 - 11:53:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 11:53:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 11:54:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.336966 43.452213 43.206295 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.445801 72.046875 69.822266 +util 98.250000 100.000000 94.000000 +[Apr 20, 2026 - 11:54:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 493 : avg iteration time 54.78 seconds +[Apr 20, 2026 - 11:54:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 493 : data loading time 0.00 +[Apr 20, 2026 - 11:54:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 493 : avg forward pass time 15.31 +[Apr 20, 2026 - 11:54:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 493 : backward pass time 11.61 +[Apr 20, 2026 - 11:54:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 493 : optimizer step time 0.97 +[Apr 20, 2026 - 11:54:18 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 493-------------------- +[Apr 20, 2026 - 11:54:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0099 iter count: 1.0 +[Apr 20, 2026 - 11:54:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0099 iter count: 1.0 +[Apr 20, 2026 - 11:54:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 11:54:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 11:55:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.337066 43.452324 43.206482 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.484863 72.203125 69.822266 +util 97.500000 100.000000 95.000000 +[Apr 20, 2026 - 11:55:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 494 : avg iteration time 58.68 seconds +[Apr 20, 2026 - 11:55:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 494 : data loading time 0.00 +[Apr 20, 2026 - 11:55:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 494 : avg forward pass time 17.25 +[Apr 20, 2026 - 11:55:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 494 : backward pass time 11.64 +[Apr 20, 2026 - 11:55:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 494 : optimizer step time 0.98 +[Apr 20, 2026 - 11:55:17 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 494-------------------- +[Apr 20, 2026 - 11:55:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0105 iter count: 1.0 +[Apr 20, 2026 - 11:55:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[Apr 20, 2026 - 11:55:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 11:55:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 11:58:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.336842 43.451229 43.206486 +peak_gpu_mem_gb 129.547819 131.549380 123.547254 +peak_gpu_mem_reserved_gb 133.541016 135.556641 127.648438 +util 69.500000 79.000000 64.000000 +[Apr 20, 2026 - 11:58:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 495 : avg iteration time 196.16 seconds +[Apr 20, 2026 - 11:58:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 495 : data loading time 0.00 +[Apr 20, 2026 - 11:58:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 495 : avg forward pass time 89.58 +[Apr 20, 2026 - 11:58:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 495 : backward pass time 8.36 +[Apr 20, 2026 - 11:58:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 495 : optimizer step time 0.33 +[Apr 20, 2026 - 11:58:33 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 495-------------------- +[Apr 20, 2026 - 11:58:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 33020.9246 iter count: 1.0 +[Apr 20, 2026 - 11:58:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6581 iter count: 1.0 +[Apr 20, 2026 - 11:58:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 33020.9246iter count: 1.0 +[Apr 20, 2026 - 11:58:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 11:58:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.4043iter count: 1.0 +[Apr 20, 2026 - 11:58:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.5059 iter count: 1.0 +[Apr 20, 2026 - 11:58:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.4217 iter count: 1.0 +[Apr 20, 2026 - 11:58:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 46143.5234iter count: 1.0 +[Apr 20, 2026 - 11:58:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 20.3253iter count: 1.0 +[Apr 20, 2026 - 11:58:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 378248.8125iter count: 1.0 +[Apr 20, 2026 - 11:58:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6581iter count: 1.0 +[Apr 20, 2026 - 11:58:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 5.5140 iter count: 1.0 +[Apr 20, 2026 - 11:58:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 61572.0703iter count: 1.0 +[Apr 20, 2026 - 11:58:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 5.1704 iter count: 1.0 +[Apr 20, 2026 - 11:58:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 30969.2402iter count: 1.0 +[Apr 20, 2026 - 11:58:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 6.0986 iter count: 1.0 +[Apr 20, 2026 - 11:58:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 198218.5469iter count: 1.0 +[Apr 20, 2026 - 11:58:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.1842 iter count: 1.0 +[Apr 20, 2026 - 11:58:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 4309.0679iter count: 1.0 +[Apr 20, 2026 - 11:58:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0095 iter count: 1.0 +[Apr 20, 2026 - 11:58:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 11:59:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.336946 43.451336 43.206470 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.509277 72.496094 69.626953 +util 96.000000 99.000000 91.000000 +[Apr 20, 2026 - 11:59:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 496 : avg iteration time 48.76 seconds +[Apr 20, 2026 - 11:59:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 496 : data loading time 0.00 +[Apr 20, 2026 - 11:59:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 496 : avg forward pass time 11.83 +[Apr 20, 2026 - 11:59:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 496 : backward pass time 11.52 +[Apr 20, 2026 - 11:59:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 496 : optimizer step time 0.97 +[Apr 20, 2026 - 11:59:22 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 496-------------------- +[Apr 20, 2026 - 11:59:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0119 iter count: 1.0 +[Apr 20, 2026 - 11:59:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[Apr 20, 2026 - 11:59:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 11:59:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 12:00:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.337295 43.452587 43.206589 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.616699 72.808594 69.626953 +util 95.000000 100.000000 91.000000 +[Apr 20, 2026 - 12:00:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 497 : avg iteration time 56.01 seconds +[Apr 20, 2026 - 12:00:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 497 : data loading time 0.00 +[Apr 20, 2026 - 12:00:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 497 : avg forward pass time 15.73 +[Apr 20, 2026 - 12:00:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 497 : backward pass time 11.97 +[Apr 20, 2026 - 12:00:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 497 : optimizer step time 0.98 +[Apr 20, 2026 - 12:00:18 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 497-------------------- +[Apr 20, 2026 - 12:00:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0097 iter count: 1.0 +[Apr 20, 2026 - 12:00:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[Apr 20, 2026 - 12:00:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 12:00:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 12:01:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.337291 43.452587 43.206566 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.602051 72.750000 69.626953 +util 97.250000 99.000000 94.000000 +[Apr 20, 2026 - 12:01:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 498 : avg iteration time 61.25 seconds +[Apr 20, 2026 - 12:01:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 498 : data loading time 0.00 +[Apr 20, 2026 - 12:01:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 498 : avg forward pass time 18.16 +[Apr 20, 2026 - 12:01:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 498 : backward pass time 11.55 +[Apr 20, 2026 - 12:01:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 498 : optimizer step time 1.07 +[Apr 20, 2026 - 12:01:19 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 498-------------------- +[Apr 20, 2026 - 12:01:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0106 iter count: 1.0 +[Apr 20, 2026 - 12:01:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[Apr 20, 2026 - 12:01:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 12:01:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 12:02:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.337303 43.452595 43.206585 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.621582 72.769531 69.724609 +util 98.000000 100.000000 93.000000 +[Apr 20, 2026 - 12:02:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 499 : avg iteration time 60.70 seconds +[Apr 20, 2026 - 12:02:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 499 : data loading time 0.00 +[Apr 20, 2026 - 12:02:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 499 : avg forward pass time 18.33 +[Apr 20, 2026 - 12:02:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 499 : backward pass time 11.54 +[Apr 20, 2026 - 12:02:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 499 : optimizer step time 0.96 +[Apr 20, 2026 - 12:02:20 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 499-------------------- +[Apr 20, 2026 - 12:02:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0102 iter count: 1.0 +[Apr 20, 2026 - 12:02:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[Apr 20, 2026 - 12:02:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 12:02:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 12:05:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.337167 43.451778 43.206608 +peak_gpu_mem_gb 129.547819 131.549380 123.547254 +peak_gpu_mem_reserved_gb 133.535645 135.537109 127.587891 +util 71.500000 75.000000 69.000000 +[Apr 20, 2026 - 12:05:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 500 : avg iteration time 179.94 seconds +[Apr 20, 2026 - 12:05:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 500 : data loading time 0.00 +[Apr 20, 2026 - 12:05:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 500 : avg forward pass time 81.40 +[Apr 20, 2026 - 12:05:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 500 : backward pass time 8.41 +[Apr 20, 2026 - 12:05:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 500 : optimizer step time 0.33 +[Apr 20, 2026 - 12:05:20 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 500-------------------- +[Apr 20, 2026 - 12:05:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 75955.0775 iter count: 1.0 +[Apr 20, 2026 - 12:05:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6186 iter count: 1.0 +[Apr 20, 2026 - 12:05:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 75955.0775iter count: 1.0 +[Apr 20, 2026 - 12:05:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 12:05:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.9021iter count: 1.0 +[Apr 20, 2026 - 12:05:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.8838 iter count: 1.0 +[Apr 20, 2026 - 12:05:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.2572 iter count: 1.0 +[Apr 20, 2026 - 12:05:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 108835.9297iter count: 1.0 +[Apr 20, 2026 - 12:05:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 5.8572 iter count: 1.0 +[Apr 20, 2026 - 12:05:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 2011587.6250iter count: 1.0 +[Apr 20, 2026 - 12:05:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6186iter count: 1.0 +[Apr 20, 2026 - 12:05:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 2.7567 iter count: 1.0 +[Apr 20, 2026 - 12:05:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 247.9917iter count: 1.0 +[Apr 20, 2026 - 12:05:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 6.8264 iter count: 1.0 +[Apr 20, 2026 - 12:05:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 849848.1875iter count: 1.0 +[Apr 20, 2026 - 12:05:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 1.6327 iter count: 1.0 +[Apr 20, 2026 - 12:05:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 26.1884 iter count: 1.0 +[Apr 20, 2026 - 12:05:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 3.1292 iter count: 1.0 +[Apr 20, 2026 - 12:05:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 522.3436iter count: 1.0 +[Apr 20, 2026 - 12:05:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[Apr 20, 2026 - 12:05:41 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 20.79 seconds +[Apr 20, 2026 - 12:05:41 | INFO | fastgen.trainer:save_checkpoint:266 ] Saving checkpoint iteration 500 +[Apr 20, 2026 - 12:05:41 | INFO | fastgen.utils.checkpointer:save:301 ] Saving FSDP model to prefix /tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000500 +[Apr 20, 2026 - 12:05:56 | SUCCESS | fastgen.utils.checkpointer:save:329 ] Model saved at iteration 500 +[Apr 20, 2026 - 12:05:56 | INFO | fastgen.trainer:validate:358 ] Validation iteration 500 +[Apr 20, 2026 - 12:05:57 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 12:06:00 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 3.4s +[Apr 20, 2026 - 12:06:04 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 12:06:07 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 2.9s +[Apr 20, 2026 - 12:06:10 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 12:06:13 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 2.9s +[Apr 20, 2026 - 12:06:16 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 12:06:19 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 2.9s +[Apr 20, 2026 - 12:06:23 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 12:06:26 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 2.9s +[Apr 20, 2026 - 12:06:29 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 12:06:32 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 2.9s +[Apr 20, 2026 - 12:06:35 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 12:06:38 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 2.9s +[Apr 20, 2026 - 12:06:41 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 12:06:44 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 2.9s +[Apr 20, 2026 - 12:06:48 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 12:06:50 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 2.9s +[Apr 20, 2026 - 12:06:54 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 12:06:57 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 2.9s +[Apr 20, 2026 - 12:07:00 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging val0 stats at iteration 500-------------------- +[Apr 20, 2026 - 12:07:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0000 iter count: 10.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[Apr 20, 2026 - 12:07:16 | INFO | fastgen.callbacks.wandb:on_validation_end:683 ] Logged 10 val videos at iteration 500 +[Apr 20, 2026 - 12:08:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338567 43.457668 43.206245 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.923340 70.349609 69.664062 +util 97.500000 100.000000 92.000000 +[Apr 20, 2026 - 12:08:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 501 : avg iteration time 170.46 seconds +[Apr 20, 2026 - 12:08:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 501 : data loading time 0.00 +[Apr 20, 2026 - 12:08:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 501 : avg forward pass time 15.10 +[Apr 20, 2026 - 12:08:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 501 : backward pass time 11.56 +[Apr 20, 2026 - 12:08:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 501 : optimizer step time 0.97 +[Apr 20, 2026 - 12:08:10 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 501-------------------- +[Apr 20, 2026 - 12:08:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0121 iter count: 1.0 +[Apr 20, 2026 - 12:08:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[Apr 20, 2026 - 12:08:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 12:08:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 12:09:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338557 43.457623 43.206264 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.025879 70.271484 69.742188 +util 98.000000 100.000000 93.000000 +[Apr 20, 2026 - 12:09:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 502 : avg iteration time 58.81 seconds +[Apr 20, 2026 - 12:09:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 502 : data loading time 0.00 +[Apr 20, 2026 - 12:09:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 502 : avg forward pass time 16.72 +[Apr 20, 2026 - 12:09:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 502 : backward pass time 11.75 +[Apr 20, 2026 - 12:09:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 502 : optimizer step time 1.50 +[Apr 20, 2026 - 12:09:09 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 502-------------------- +[Apr 20, 2026 - 12:09:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0082 iter count: 1.0 +[Apr 20, 2026 - 12:09:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0082 iter count: 1.0 +[Apr 20, 2026 - 12:09:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 12:09:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 12:10:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338239 43.456161 43.206257 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.894043 70.408203 69.488281 +util 97.750000 100.000000 92.000000 +[Apr 20, 2026 - 12:10:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 503 : avg iteration time 63.19 seconds +[Apr 20, 2026 - 12:10:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 503 : data loading time 0.00 +[Apr 20, 2026 - 12:10:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 503 : avg forward pass time 19.52 +[Apr 20, 2026 - 12:10:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 503 : backward pass time 11.59 +[Apr 20, 2026 - 12:10:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 503 : optimizer step time 0.97 +[Apr 20, 2026 - 12:10:12 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 503-------------------- +[Apr 20, 2026 - 12:10:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0101 iter count: 1.0 +[Apr 20, 2026 - 12:10:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[Apr 20, 2026 - 12:10:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 12:10:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 12:11:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338214 43.456123 43.206257 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.859863 70.212891 69.468750 +util 96.000000 99.000000 91.000000 +[Apr 20, 2026 - 12:11:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 504 : avg iteration time 51.05 seconds +[Apr 20, 2026 - 12:11:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 504 : data loading time 0.00 +[Apr 20, 2026 - 12:11:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 504 : avg forward pass time 12.84 +[Apr 20, 2026 - 12:11:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 504 : backward pass time 12.59 +[Apr 20, 2026 - 12:11:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 504 : optimizer step time 1.22 +[Apr 20, 2026 - 12:11:04 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 504-------------------- +[Apr 20, 2026 - 12:11:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0094 iter count: 1.0 +[Apr 20, 2026 - 12:11:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[Apr 20, 2026 - 12:11:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 12:11:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 12:14:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.337738 43.452812 43.206726 +peak_gpu_mem_gb 129.547819 131.549380 123.547254 +peak_gpu_mem_reserved_gb 133.682617 135.615234 128.058594 +util 73.250000 78.000000 64.000000 +[Apr 20, 2026 - 12:14:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 505 : avg iteration time 188.77 seconds +[Apr 20, 2026 - 12:14:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 505 : data loading time 0.00 +[Apr 20, 2026 - 12:14:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 505 : avg forward pass time 85.87 +[Apr 20, 2026 - 12:14:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 505 : backward pass time 8.36 +[Apr 20, 2026 - 12:14:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 505 : optimizer step time 0.33 +[Apr 20, 2026 - 12:14:12 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 505-------------------- +[Apr 20, 2026 - 12:14:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 53622.1924 iter count: 1.0 +[Apr 20, 2026 - 12:14:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6680 iter count: 1.0 +[Apr 20, 2026 - 12:14:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 53622.1924iter count: 1.0 +[Apr 20, 2026 - 12:14:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 12:14:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.4748iter count: 1.0 +[Apr 20, 2026 - 12:14:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 2.0179 iter count: 1.0 +[Apr 20, 2026 - 12:14:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.6648 iter count: 1.0 +[Apr 20, 2026 - 12:14:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 75563.9219iter count: 1.0 +[Apr 20, 2026 - 12:14:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 56.5853iter count: 1.0 +[Apr 20, 2026 - 12:14:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 615077.7500iter count: 1.0 +[Apr 20, 2026 - 12:14:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6680iter count: 1.0 +[Apr 20, 2026 - 12:14:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.6629 iter count: 1.0 +[Apr 20, 2026 - 12:14:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 1518.8668iter count: 1.0 +[Apr 20, 2026 - 12:14:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.3590 iter count: 1.0 +[Apr 20, 2026 - 12:14:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 6112.4819iter count: 1.0 +[Apr 20, 2026 - 12:14:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 3.2253 iter count: 1.0 +[Apr 20, 2026 - 12:14:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 633.0697iter count: 1.0 +[Apr 20, 2026 - 12:14:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 2.0179 iter count: 1.0 +[Apr 20, 2026 - 12:14:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 56.5853 iter count: 1.0 +[Apr 20, 2026 - 12:14:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0078 iter count: 1.0 +[Apr 20, 2026 - 12:14:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 12:15:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.337841 43.453236 43.206715 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.606934 72.634766 69.898438 +util 97.500000 100.000000 92.000000 +[Apr 20, 2026 - 12:15:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 506 : avg iteration time 54.38 seconds +[Apr 20, 2026 - 12:15:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 506 : data loading time 0.00 +[Apr 20, 2026 - 12:15:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 506 : avg forward pass time 14.73 +[Apr 20, 2026 - 12:15:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 506 : backward pass time 11.53 +[Apr 20, 2026 - 12:15:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 506 : optimizer step time 1.01 +[Apr 20, 2026 - 12:15:07 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 506-------------------- +[Apr 20, 2026 - 12:15:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0106 iter count: 1.0 +[Apr 20, 2026 - 12:15:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[Apr 20, 2026 - 12:15:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 12:15:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 12:16:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338031 43.453995 43.206715 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.947754 70.035156 69.822266 +util 97.750000 100.000000 93.000000 +[Apr 20, 2026 - 12:16:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 507 : avg iteration time 61.02 seconds +[Apr 20, 2026 - 12:16:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 507 : data loading time 0.00 +[Apr 20, 2026 - 12:16:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 507 : avg forward pass time 18.46 +[Apr 20, 2026 - 12:16:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 507 : backward pass time 11.56 +[Apr 20, 2026 - 12:16:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 507 : optimizer step time 0.99 +[Apr 20, 2026 - 12:16:08 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 507-------------------- +[Apr 20, 2026 - 12:16:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0124 iter count: 1.0 +[Apr 20, 2026 - 12:16:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0124 iter count: 1.0 +[Apr 20, 2026 - 12:16:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 12:16:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 12:17:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.337989 43.453789 43.206741 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.957520 70.054688 69.841797 +util 94.000000 98.000000 91.000000 +[Apr 20, 2026 - 12:17:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 508 : avg iteration time 62.24 seconds +[Apr 20, 2026 - 12:17:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 508 : data loading time 0.00 +[Apr 20, 2026 - 12:17:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 508 : avg forward pass time 18.29 +[Apr 20, 2026 - 12:17:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 508 : backward pass time 12.92 +[Apr 20, 2026 - 12:17:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 508 : optimizer step time 1.16 +[Apr 20, 2026 - 12:17:10 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 508-------------------- +[Apr 20, 2026 - 12:17:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0118 iter count: 1.0 +[Apr 20, 2026 - 12:17:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0118 iter count: 1.0 +[Apr 20, 2026 - 12:17:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 12:17:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 12:18:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338016 43.453789 43.206722 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.894043 70.074219 69.742188 +util 99.000000 100.000000 98.000000 +[Apr 20, 2026 - 12:18:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 509 : avg iteration time 54.48 seconds +[Apr 20, 2026 - 12:18:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 509 : data loading time 0.00 +[Apr 20, 2026 - 12:18:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 509 : avg forward pass time 15.19 +[Apr 20, 2026 - 12:18:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 509 : backward pass time 11.56 +[Apr 20, 2026 - 12:18:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 509 : optimizer step time 0.98 +[Apr 20, 2026 - 12:18:04 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 509-------------------- +[Apr 20, 2026 - 12:18:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0094 iter count: 1.0 +[Apr 20, 2026 - 12:18:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0094 iter count: 1.0 +[Apr 20, 2026 - 12:18:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 12:18:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 12:21:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.337742 43.452698 43.206734 +peak_gpu_mem_gb 129.547819 131.549380 123.547254 +peak_gpu_mem_reserved_gb 133.613770 135.537109 127.902344 +util 70.250000 80.000000 53.000000 +[Apr 20, 2026 - 12:21:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 510 : avg iteration time 190.03 seconds +[Apr 20, 2026 - 12:21:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 510 : data loading time 0.00 +[Apr 20, 2026 - 12:21:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 510 : avg forward pass time 86.27 +[Apr 20, 2026 - 12:21:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 510 : backward pass time 8.83 +[Apr 20, 2026 - 12:21:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 510 : optimizer step time 0.33 +[Apr 20, 2026 - 12:21:14 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 510-------------------- +[Apr 20, 2026 - 12:21:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 82372.8027 iter count: 1.0 +[Apr 20, 2026 - 12:21:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6064 iter count: 1.0 +[Apr 20, 2026 - 12:21:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 82372.8027iter count: 1.0 +[Apr 20, 2026 - 12:21:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 12:21:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.4283iter count: 1.0 +[Apr 20, 2026 - 12:21:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.7056 iter count: 1.0 +[Apr 20, 2026 - 12:21:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.3173 iter count: 1.0 +[Apr 20, 2026 - 12:21:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 163394.1406iter count: 1.0 +[Apr 20, 2026 - 12:21:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 4.1011 iter count: 1.0 +[Apr 20, 2026 - 12:21:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 2268238.0000iter count: 1.0 +[Apr 20, 2026 - 12:21:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6064iter count: 1.0 +[Apr 20, 2026 - 12:21:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.4339 iter count: 1.0 +[Apr 20, 2026 - 12:21:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 960.9275iter count: 1.0 +[Apr 20, 2026 - 12:21:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.2766 iter count: 1.0 +[Apr 20, 2026 - 12:21:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 5183.6001iter count: 1.0 +[Apr 20, 2026 - 12:21:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 5.6212 iter count: 1.0 +[Apr 20, 2026 - 12:21:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 76297.9375iter count: 1.0 +[Apr 20, 2026 - 12:21:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 3.5452 iter count: 1.0 +[Apr 20, 2026 - 12:21:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 1200.4276iter count: 1.0 +[Apr 20, 2026 - 12:21:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[Apr 20, 2026 - 12:21:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 12:22:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.337816 43.452957 43.206757 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.923340 69.996094 69.822266 +util 97.000000 100.000000 94.000000 +[Apr 20, 2026 - 12:22:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 511 : avg iteration time 60.53 seconds +[Apr 20, 2026 - 12:22:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 511 : data loading time 0.00 +[Apr 20, 2026 - 12:22:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 511 : avg forward pass time 17.68 +[Apr 20, 2026 - 12:22:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 511 : backward pass time 12.55 +[Apr 20, 2026 - 12:22:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 511 : optimizer step time 1.02 +[Apr 20, 2026 - 12:22:15 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 511-------------------- +[Apr 20, 2026 - 12:22:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0106 iter count: 1.0 +[Apr 20, 2026 - 12:22:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[Apr 20, 2026 - 12:22:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 12:22:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 12:23:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338082 43.453957 43.206741 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.815918 69.976562 69.527344 +util 99.000000 99.000000 99.000000 +[Apr 20, 2026 - 12:23:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 512 : avg iteration time 58.06 seconds +[Apr 20, 2026 - 12:23:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 512 : data loading time 0.00 +[Apr 20, 2026 - 12:23:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 512 : avg forward pass time 16.90 +[Apr 20, 2026 - 12:23:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 512 : backward pass time 11.54 +[Apr 20, 2026 - 12:23:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 512 : optimizer step time 0.99 +[Apr 20, 2026 - 12:23:13 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 512-------------------- +[Apr 20, 2026 - 12:23:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0088 iter count: 1.0 +[Apr 20, 2026 - 12:23:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0088 iter count: 1.0 +[Apr 20, 2026 - 12:23:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 12:23:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 12:24:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338132 43.454018 43.206772 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.825684 69.976562 69.488281 +util 97.500000 100.000000 92.000000 +[Apr 20, 2026 - 12:24:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 513 : avg iteration time 56.83 seconds +[Apr 20, 2026 - 12:24:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 513 : data loading time 0.00 +[Apr 20, 2026 - 12:24:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 513 : avg forward pass time 15.71 +[Apr 20, 2026 - 12:24:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 513 : backward pass time 12.81 +[Apr 20, 2026 - 12:24:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 513 : optimizer step time 1.05 +[Apr 20, 2026 - 12:24:10 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 513-------------------- +[Apr 20, 2026 - 12:24:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0098 iter count: 1.0 +[Apr 20, 2026 - 12:24:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[Apr 20, 2026 - 12:24:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 12:24:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 12:25:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338138 43.454018 43.206776 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.913574 69.957031 69.878906 +util 99.500000 100.000000 99.000000 +[Apr 20, 2026 - 12:25:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 514 : avg iteration time 52.18 seconds +[Apr 20, 2026 - 12:25:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 514 : data loading time 0.00 +[Apr 20, 2026 - 12:25:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 514 : avg forward pass time 13.98 +[Apr 20, 2026 - 12:25:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 514 : backward pass time 11.55 +[Apr 20, 2026 - 12:25:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 514 : optimizer step time 0.99 +[Apr 20, 2026 - 12:25:02 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 514-------------------- +[Apr 20, 2026 - 12:25:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0097 iter count: 1.0 +[Apr 20, 2026 - 12:25:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[Apr 20, 2026 - 12:25:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 12:25:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 12:28:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.337955 43.453239 43.206772 +peak_gpu_mem_gb 129.547819 131.549380 123.547254 +peak_gpu_mem_reserved_gb 133.554688 135.554688 127.609375 +util 71.500000 79.000000 63.000000 +[Apr 20, 2026 - 12:28:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 515 : avg iteration time 192.27 seconds +[Apr 20, 2026 - 12:28:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 515 : data loading time 0.00 +[Apr 20, 2026 - 12:28:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 515 : avg forward pass time 86.99 +[Apr 20, 2026 - 12:28:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 515 : backward pass time 8.34 +[Apr 20, 2026 - 12:28:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 515 : optimizer step time 0.33 +[Apr 20, 2026 - 12:28:14 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 515-------------------- +[Apr 20, 2026 - 12:28:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 84534.8267 iter count: 1.0 +[Apr 20, 2026 - 12:28:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6559 iter count: 1.0 +[Apr 20, 2026 - 12:28:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 84534.8267iter count: 1.0 +[Apr 20, 2026 - 12:28:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 12:28:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.1244iter count: 1.0 +[Apr 20, 2026 - 12:28:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.2306 iter count: 1.0 +[Apr 20, 2026 - 12:28:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.4272 iter count: 1.0 +[Apr 20, 2026 - 12:28:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 149510.7656iter count: 1.0 +[Apr 20, 2026 - 12:28:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 11.7195iter count: 1.0 +[Apr 20, 2026 - 12:28:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 2826243.0000iter count: 1.0 +[Apr 20, 2026 - 12:28:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6559iter count: 1.0 +[Apr 20, 2026 - 12:28:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 6.7679 iter count: 1.0 +[Apr 20, 2026 - 12:28:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 755962.8750iter count: 1.0 +[Apr 20, 2026 - 12:28:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 5.5283 iter count: 1.0 +[Apr 20, 2026 - 12:28:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 63354.6328iter count: 1.0 +[Apr 20, 2026 - 12:28:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 3.9573 iter count: 1.0 +[Apr 20, 2026 - 12:28:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 2737.1560iter count: 1.0 +[Apr 20, 2026 - 12:28:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 3.6149 iter count: 1.0 +[Apr 20, 2026 - 12:28:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 1379.9351iter count: 1.0 +[Apr 20, 2026 - 12:28:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[Apr 20, 2026 - 12:28:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 12:29:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.337970 43.453236 43.206795 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.820801 70.035156 69.470703 +util 97.250000 100.000000 91.000000 +[Apr 20, 2026 - 12:29:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 516 : avg iteration time 52.54 seconds +[Apr 20, 2026 - 12:29:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 516 : data loading time 0.00 +[Apr 20, 2026 - 12:29:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 516 : avg forward pass time 13.65 +[Apr 20, 2026 - 12:29:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 516 : backward pass time 11.54 +[Apr 20, 2026 - 12:29:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 516 : optimizer step time 0.98 +[Apr 20, 2026 - 12:29:07 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 516-------------------- +[Apr 20, 2026 - 12:29:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0109 iter count: 1.0 +[Apr 20, 2026 - 12:29:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0109 iter count: 1.0 +[Apr 20, 2026 - 12:29:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 12:29:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 12:30:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338035 43.453480 43.206806 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.815918 70.035156 69.451172 +util 97.750000 100.000000 92.000000 +[Apr 20, 2026 - 12:30:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 517 : avg iteration time 56.81 seconds +[Apr 20, 2026 - 12:30:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 517 : data loading time 0.00 +[Apr 20, 2026 - 12:30:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 517 : avg forward pass time 15.50 +[Apr 20, 2026 - 12:30:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 517 : backward pass time 12.22 +[Apr 20, 2026 - 12:30:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 517 : optimizer step time 1.59 +[Apr 20, 2026 - 12:30:04 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 517-------------------- +[Apr 20, 2026 - 12:30:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0102 iter count: 1.0 +[Apr 20, 2026 - 12:30:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[Apr 20, 2026 - 12:30:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 12:30:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 12:31:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338044 43.453487 43.206841 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.859863 69.937500 69.822266 +util 99.750000 100.000000 99.000000 +[Apr 20, 2026 - 12:31:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 518 : avg iteration time 58.18 seconds +[Apr 20, 2026 - 12:31:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 518 : data loading time 0.00 +[Apr 20, 2026 - 12:31:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 518 : avg forward pass time 17.03 +[Apr 20, 2026 - 12:31:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 518 : backward pass time 11.55 +[Apr 20, 2026 - 12:31:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 518 : optimizer step time 0.99 +[Apr 20, 2026 - 12:31:02 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 518-------------------- +[Apr 20, 2026 - 12:31:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0090 iter count: 1.0 +[Apr 20, 2026 - 12:31:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[Apr 20, 2026 - 12:31:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 12:31:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 12:32:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338059 43.453503 43.206863 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.908691 69.976562 69.839844 +util 98.000000 100.000000 93.000000 +[Apr 20, 2026 - 12:32:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 519 : avg iteration time 62.23 seconds +[Apr 20, 2026 - 12:32:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 519 : data loading time 0.00 +[Apr 20, 2026 - 12:32:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 519 : avg forward pass time 18.00 +[Apr 20, 2026 - 12:32:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 519 : backward pass time 12.14 +[Apr 20, 2026 - 12:32:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 519 : optimizer step time 1.56 +[Apr 20, 2026 - 12:32:04 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 519-------------------- +[Apr 20, 2026 - 12:32:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0093 iter count: 1.0 +[Apr 20, 2026 - 12:32:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[Apr 20, 2026 - 12:32:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 12:32:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 12:35:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.337701 43.452068 43.206886 +peak_gpu_mem_gb 129.547819 131.549380 123.547254 +peak_gpu_mem_reserved_gb 133.491699 135.537109 127.451172 +util 72.750000 76.000000 70.000000 +[Apr 20, 2026 - 12:35:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 520 : avg iteration time 181.76 seconds +[Apr 20, 2026 - 12:35:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 520 : data loading time 0.00 +[Apr 20, 2026 - 12:35:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 520 : avg forward pass time 82.37 +[Apr 20, 2026 - 12:35:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 520 : backward pass time 8.36 +[Apr 20, 2026 - 12:35:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 520 : optimizer step time 0.34 +[Apr 20, 2026 - 12:35:06 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 520-------------------- +[Apr 20, 2026 - 12:35:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 120900.4434 iter count: 1.0 +[Apr 20, 2026 - 12:35:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6182 iter count: 1.0 +[Apr 20, 2026 - 12:35:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 120900.4434iter count: 1.0 +[Apr 20, 2026 - 12:35:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 12:35:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.2953iter count: 1.0 +[Apr 20, 2026 - 12:35:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.6807 iter count: 1.0 +[Apr 20, 2026 - 12:35:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.3699 iter count: 1.0 +[Apr 20, 2026 - 12:35:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 238370.3438iter count: 1.0 +[Apr 20, 2026 - 12:35:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 3.9020 iter count: 1.0 +[Apr 20, 2026 - 12:35:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 2520311.2500iter count: 1.0 +[Apr 20, 2026 - 12:35:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6182iter count: 1.0 +[Apr 20, 2026 - 12:35:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 1.2601 iter count: 1.0 +[Apr 20, 2026 - 12:35:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 12.4311 iter count: 1.0 +[Apr 20, 2026 - 12:35:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.4976 iter count: 1.0 +[Apr 20, 2026 - 12:35:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 8064.4604iter count: 1.0 +[Apr 20, 2026 - 12:35:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.5979 iter count: 1.0 +[Apr 20, 2026 - 12:35:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 9854.9795iter count: 1.0 +[Apr 20, 2026 - 12:35:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 6.3373 iter count: 1.0 +[Apr 20, 2026 - 12:35:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 319531.4688iter count: 1.0 +[Apr 20, 2026 - 12:35:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0097 iter count: 1.0 +[Apr 20, 2026 - 12:35:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 12:36:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.337851 43.452564 43.206959 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.874512 69.957031 69.742188 +util 98.000000 100.000000 96.000000 +[Apr 20, 2026 - 12:36:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 521 : avg iteration time 61.36 seconds +[Apr 20, 2026 - 12:36:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 521 : data loading time 0.00 +[Apr 20, 2026 - 12:36:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 521 : avg forward pass time 17.69 +[Apr 20, 2026 - 12:36:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 521 : backward pass time 13.10 +[Apr 20, 2026 - 12:36:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 521 : optimizer step time 1.24 +[Apr 20, 2026 - 12:36:07 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 521-------------------- +[Apr 20, 2026 - 12:36:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0129 iter count: 1.0 +[Apr 20, 2026 - 12:36:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0129 iter count: 1.0 +[Apr 20, 2026 - 12:36:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 12:36:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 12:37:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338299 43.454292 43.206985 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.786621 70.035156 69.470703 +util 97.500000 100.000000 92.000000 +[Apr 20, 2026 - 12:37:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 522 : avg iteration time 60.87 seconds +[Apr 20, 2026 - 12:37:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 522 : data loading time 0.01 +[Apr 20, 2026 - 12:37:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 522 : avg forward pass time 18.41 +[Apr 20, 2026 - 12:37:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 522 : backward pass time 11.55 +[Apr 20, 2026 - 12:37:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 522 : optimizer step time 0.97 +[Apr 20, 2026 - 12:37:08 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 522-------------------- +[Apr 20, 2026 - 12:37:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0105 iter count: 1.0 +[Apr 20, 2026 - 12:37:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[Apr 20, 2026 - 12:37:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 12:37:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 12:37:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338313 43.454296 43.207081 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.752441 70.054688 69.468750 +util 97.750000 99.000000 94.000000 +[Apr 20, 2026 - 12:37:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 523 : avg iteration time 47.62 seconds +[Apr 20, 2026 - 12:37:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 523 : data loading time 0.00 +[Apr 20, 2026 - 12:37:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 523 : avg forward pass time 11.35 +[Apr 20, 2026 - 12:37:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 523 : backward pass time 11.63 +[Apr 20, 2026 - 12:37:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 523 : optimizer step time 1.10 +[Apr 20, 2026 - 12:37:56 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 523-------------------- +[Apr 20, 2026 - 12:37:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0100 iter count: 1.0 +[Apr 20, 2026 - 12:37:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[Apr 20, 2026 - 12:37:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 12:37:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 12:38:49 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338387 43.454544 43.207081 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.864746 70.054688 69.468750 +util 98.000000 100.000000 94.000000 +[Apr 20, 2026 - 12:38:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 524 : avg iteration time 52.83 seconds +[Apr 20, 2026 - 12:38:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 524 : data loading time 0.00 +[Apr 20, 2026 - 12:38:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 524 : avg forward pass time 14.27 +[Apr 20, 2026 - 12:38:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 524 : backward pass time 11.70 +[Apr 20, 2026 - 12:38:49 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 524 : optimizer step time 1.02 +[Apr 20, 2026 - 12:38:49 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 524-------------------- +[Apr 20, 2026 - 12:38:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0089 iter count: 1.0 +[Apr 20, 2026 - 12:38:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[Apr 20, 2026 - 12:38:49 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 12:38:49 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 12:42:03 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338139 43.453411 43.207092 +peak_gpu_mem_gb 129.547819 131.549380 123.547254 +peak_gpu_mem_reserved_gb 133.550293 135.597656 127.509766 +util 72.250000 77.000000 63.000000 +[Apr 20, 2026 - 12:42:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 525 : avg iteration time 194.37 seconds +[Apr 20, 2026 - 12:42:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 525 : data loading time 0.00 +[Apr 20, 2026 - 12:42:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 525 : avg forward pass time 88.65 +[Apr 20, 2026 - 12:42:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 525 : backward pass time 8.39 +[Apr 20, 2026 - 12:42:03 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 525 : optimizer step time 0.34 +[Apr 20, 2026 - 12:42:03 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 525-------------------- +[Apr 20, 2026 - 12:42:03 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 35770.8123 iter count: 1.0 +[Apr 20, 2026 - 12:42:03 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6735 iter count: 1.0 +[Apr 20, 2026 - 12:42:03 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 35770.8123iter count: 1.0 +[Apr 20, 2026 - 12:42:03 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 12:42:03 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.8833iter count: 1.0 +[Apr 20, 2026 - 12:42:03 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.5623 iter count: 1.0 +[Apr 20, 2026 - 12:42:03 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.6415 iter count: 1.0 +[Apr 20, 2026 - 12:42:03 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 52900.2227iter count: 1.0 +[Apr 20, 2026 - 12:42:03 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 3.0788 iter count: 1.0 +[Apr 20, 2026 - 12:42:03 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 587170.5000iter count: 1.0 +[Apr 20, 2026 - 12:42:03 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6735iter count: 1.0 +[Apr 20, 2026 - 12:42:03 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 0.5623 iter count: 1.0 +[Apr 20, 2026 - 12:42:03 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 3.0788 iter count: 1.0 +[Apr 20, 2026 - 12:42:03 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 5.2612 iter count: 1.0 +[Apr 20, 2026 - 12:42:03 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 37140.0586iter count: 1.0 +[Apr 20, 2026 - 12:42:03 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 6.3644 iter count: 1.0 +[Apr 20, 2026 - 12:42:03 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 337352.5312iter count: 1.0 +[Apr 20, 2026 - 12:42:03 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 3.8611 iter count: 1.0 +[Apr 20, 2026 - 12:42:03 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 2257.7458iter count: 1.0 +[Apr 20, 2026 - 12:42:03 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0089 iter count: 1.0 +[Apr 20, 2026 - 12:42:03 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 12:43:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338202 43.453659 43.207092 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.937988 70.074219 69.822266 +util 99.250000 100.000000 99.000000 +[Apr 20, 2026 - 12:43:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 526 : avg iteration time 57.35 seconds +[Apr 20, 2026 - 12:43:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 526 : data loading time 0.00 +[Apr 20, 2026 - 12:43:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 526 : avg forward pass time 16.25 +[Apr 20, 2026 - 12:43:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 526 : backward pass time 11.56 +[Apr 20, 2026 - 12:43:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 526 : optimizer step time 0.99 +[Apr 20, 2026 - 12:43:00 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 526-------------------- +[Apr 20, 2026 - 12:43:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0101 iter count: 1.0 +[Apr 20, 2026 - 12:43:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[Apr 20, 2026 - 12:43:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 12:43:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 12:44:02 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338226 43.453659 43.207214 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.874512 70.015625 69.705078 +util 98.750000 100.000000 97.000000 +[Apr 20, 2026 - 12:44:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 527 : avg iteration time 61.98 seconds +[Apr 20, 2026 - 12:44:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 527 : data loading time 0.00 +[Apr 20, 2026 - 12:44:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 527 : avg forward pass time 18.76 +[Apr 20, 2026 - 12:44:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 527 : backward pass time 11.93 +[Apr 20, 2026 - 12:44:02 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 527 : optimizer step time 0.98 +[Apr 20, 2026 - 12:44:02 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 527-------------------- +[Apr 20, 2026 - 12:44:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0111 iter count: 1.0 +[Apr 20, 2026 - 12:44:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[Apr 20, 2026 - 12:44:02 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 12:44:02 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 12:45:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338232 43.453659 43.207199 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.884277 70.035156 69.626953 +util 95.500000 100.000000 92.000000 +[Apr 20, 2026 - 12:45:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 528 : avg iteration time 62.36 seconds +[Apr 20, 2026 - 12:45:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 528 : data loading time 0.00 +[Apr 20, 2026 - 12:45:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 528 : avg forward pass time 18.20 +[Apr 20, 2026 - 12:45:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 528 : backward pass time 12.82 +[Apr 20, 2026 - 12:45:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 528 : optimizer step time 1.30 +[Apr 20, 2026 - 12:45:05 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 528-------------------- +[Apr 20, 2026 - 12:45:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0112 iter count: 1.0 +[Apr 20, 2026 - 12:45:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[Apr 20, 2026 - 12:45:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 12:45:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 12:46:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338358 43.454151 43.207199 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.884277 69.898438 69.861328 +util 94.750000 99.000000 92.000000 +[Apr 20, 2026 - 12:46:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 529 : avg iteration time 60.97 seconds +[Apr 20, 2026 - 12:46:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 529 : data loading time 0.00 +[Apr 20, 2026 - 12:46:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 529 : avg forward pass time 18.44 +[Apr 20, 2026 - 12:46:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 529 : backward pass time 11.56 +[Apr 20, 2026 - 12:46:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 529 : optimizer step time 0.99 +[Apr 20, 2026 - 12:46:06 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 529-------------------- +[Apr 20, 2026 - 12:46:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0111 iter count: 1.0 +[Apr 20, 2026 - 12:46:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[Apr 20, 2026 - 12:46:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 12:46:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 12:49:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.337985 43.452690 43.207199 +peak_gpu_mem_gb 129.547819 131.549380 123.547254 +peak_gpu_mem_reserved_gb 133.564941 135.574219 127.707031 +util 69.250000 73.000000 67.000000 +[Apr 20, 2026 - 12:49:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 530 : avg iteration time 179.66 seconds +[Apr 20, 2026 - 12:49:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 530 : data loading time 0.00 +[Apr 20, 2026 - 12:49:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 530 : avg forward pass time 81.22 +[Apr 20, 2026 - 12:49:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 530 : backward pass time 8.38 +[Apr 20, 2026 - 12:49:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 530 : optimizer step time 0.34 +[Apr 20, 2026 - 12:49:05 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 530-------------------- +[Apr 20, 2026 - 12:49:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 31732.2988 iter count: 1.0 +[Apr 20, 2026 - 12:49:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6497 iter count: 1.0 +[Apr 20, 2026 - 12:49:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 31732.2988iter count: 1.0 +[Apr 20, 2026 - 12:49:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 12:49:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.2925iter count: 1.0 +[Apr 20, 2026 - 12:49:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 2.3183 iter count: 1.0 +[Apr 20, 2026 - 12:49:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.5259 iter count: 1.0 +[Apr 20, 2026 - 12:49:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 52924.8984iter count: 1.0 +[Apr 20, 2026 - 12:49:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 103.1939iter count: 1.0 +[Apr 20, 2026 - 12:49:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 465944.0000iter count: 1.0 +[Apr 20, 2026 - 12:49:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6497iter count: 1.0 +[Apr 20, 2026 - 12:49:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 2.7820 iter count: 1.0 +[Apr 20, 2026 - 12:49:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 260.8670iter count: 1.0 +[Apr 20, 2026 - 12:49:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.6641 iter count: 1.0 +[Apr 20, 2026 - 12:49:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 11251.0469iter count: 1.0 +[Apr 20, 2026 - 12:49:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.8343 iter count: 1.0 +[Apr 20, 2026 - 12:49:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 15812.1709iter count: 1.0 +[Apr 20, 2026 - 12:49:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 5.0922 iter count: 1.0 +[Apr 20, 2026 - 12:49:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 26486.5664iter count: 1.0 +[Apr 20, 2026 - 12:49:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[Apr 20, 2026 - 12:49:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 12:50:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338023 43.452671 43.207203 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.519043 72.498047 69.761719 +util 97.500000 100.000000 92.000000 +[Apr 20, 2026 - 12:50:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 531 : avg iteration time 54.47 seconds +[Apr 20, 2026 - 12:50:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 531 : data loading time 0.00 +[Apr 20, 2026 - 12:50:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 531 : avg forward pass time 14.68 +[Apr 20, 2026 - 12:50:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 531 : backward pass time 11.55 +[Apr 20, 2026 - 12:50:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 531 : optimizer step time 0.97 +[Apr 20, 2026 - 12:50:00 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 531-------------------- +[Apr 20, 2026 - 12:50:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0112 iter count: 1.0 +[Apr 20, 2026 - 12:50:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[Apr 20, 2026 - 12:50:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 12:50:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 12:50:48 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338333 43.453918 43.207203 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.582520 72.537109 69.781250 +util 97.250000 100.000000 95.000000 +[Apr 20, 2026 - 12:50:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 532 : avg iteration time 48.72 seconds +[Apr 20, 2026 - 12:50:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 532 : data loading time 0.00 +[Apr 20, 2026 - 12:50:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 532 : avg forward pass time 11.60 +[Apr 20, 2026 - 12:50:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 532 : backward pass time 12.43 +[Apr 20, 2026 - 12:50:48 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 532 : optimizer step time 1.53 +[Apr 20, 2026 - 12:50:48 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 532-------------------- +[Apr 20, 2026 - 12:50:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0106 iter count: 1.0 +[Apr 20, 2026 - 12:50:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[Apr 20, 2026 - 12:50:48 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 12:50:48 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 12:51:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338333 43.453938 43.207203 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.460449 72.302734 69.761719 +util 95.500000 100.000000 91.000000 +[Apr 20, 2026 - 12:51:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 533 : avg iteration time 47.94 seconds +[Apr 20, 2026 - 12:51:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 533 : data loading time 0.00 +[Apr 20, 2026 - 12:51:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 533 : avg forward pass time 11.32 +[Apr 20, 2026 - 12:51:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 533 : backward pass time 12.66 +[Apr 20, 2026 - 12:51:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 533 : optimizer step time 1.03 +[Apr 20, 2026 - 12:51:36 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 533-------------------- +[Apr 20, 2026 - 12:51:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0100 iter count: 1.0 +[Apr 20, 2026 - 12:51:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[Apr 20, 2026 - 12:51:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 12:51:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 12:52:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338346 43.453960 43.207203 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.562988 72.322266 69.859375 +util 93.750000 98.000000 91.000000 +[Apr 20, 2026 - 12:52:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 534 : avg iteration time 52.61 seconds +[Apr 20, 2026 - 12:52:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 534 : data loading time 0.00 +[Apr 20, 2026 - 12:52:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 534 : avg forward pass time 14.24 +[Apr 20, 2026 - 12:52:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 534 : backward pass time 11.60 +[Apr 20, 2026 - 12:52:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 534 : optimizer step time 0.98 +[Apr 20, 2026 - 12:52:29 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 534-------------------- +[Apr 20, 2026 - 12:52:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0096 iter count: 1.0 +[Apr 20, 2026 - 12:52:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[Apr 20, 2026 - 12:52:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 12:52:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 12:55:42 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338157 43.453171 43.207203 +peak_gpu_mem_gb 129.547819 131.549380 123.547254 +peak_gpu_mem_reserved_gb 133.599121 135.554688 127.824219 +util 70.500000 77.000000 63.000000 +[Apr 20, 2026 - 12:55:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 535 : avg iteration time 192.58 seconds +[Apr 20, 2026 - 12:55:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 535 : data loading time 0.00 +[Apr 20, 2026 - 12:55:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 535 : avg forward pass time 87.79 +[Apr 20, 2026 - 12:55:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 535 : backward pass time 8.38 +[Apr 20, 2026 - 12:55:42 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 535 : optimizer step time 0.33 +[Apr 20, 2026 - 12:55:42 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 535-------------------- +[Apr 20, 2026 - 12:55:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 70337.0090 iter count: 1.0 +[Apr 20, 2026 - 12:55:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6230 iter count: 1.0 +[Apr 20, 2026 - 12:55:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 70337.0090iter count: 1.0 +[Apr 20, 2026 - 12:55:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 12:55:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.7331iter count: 1.0 +[Apr 20, 2026 - 12:55:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.7741 iter count: 1.0 +[Apr 20, 2026 - 12:55:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.0371 iter count: 1.0 +[Apr 20, 2026 - 12:55:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 97955.3125iter count: 1.0 +[Apr 20, 2026 - 12:55:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 4.7028 iter count: 1.0 +[Apr 20, 2026 - 12:55:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 1295102.6250iter count: 1.0 +[Apr 20, 2026 - 12:55:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6230iter count: 1.0 +[Apr 20, 2026 - 12:55:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 6.6626 iter count: 1.0 +[Apr 20, 2026 - 12:55:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 612495.7500iter count: 1.0 +[Apr 20, 2026 - 12:55:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 1.9126 iter count: 1.0 +[Apr 20, 2026 - 12:55:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 45.8444 iter count: 1.0 +[Apr 20, 2026 - 12:55:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.6702 iter count: 1.0 +[Apr 20, 2026 - 12:55:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 11390.0566iter count: 1.0 +[Apr 20, 2026 - 12:55:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 6.6476 iter count: 1.0 +[Apr 20, 2026 - 12:55:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 594358.2500iter count: 1.0 +[Apr 20, 2026 - 12:55:42 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0118 iter count: 1.0 +[Apr 20, 2026 - 12:55:42 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 12:56:34 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338267 43.453640 43.207211 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.874512 70.015625 69.666016 +util 98.250000 99.000000 97.000000 +[Apr 20, 2026 - 12:56:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 536 : avg iteration time 52.49 seconds +[Apr 20, 2026 - 12:56:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 536 : data loading time 0.00 +[Apr 20, 2026 - 12:56:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 536 : avg forward pass time 13.57 +[Apr 20, 2026 - 12:56:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 536 : backward pass time 11.56 +[Apr 20, 2026 - 12:56:34 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 536 : optimizer step time 0.98 +[Apr 20, 2026 - 12:56:34 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 536-------------------- +[Apr 20, 2026 - 12:56:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0111 iter count: 1.0 +[Apr 20, 2026 - 12:56:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[Apr 20, 2026 - 12:56:34 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 12:56:34 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 12:57:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338285 43.453663 43.207233 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.947754 70.015625 69.841797 +util 97.250000 100.000000 91.000000 +[Apr 20, 2026 - 12:57:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 537 : avg iteration time 56.86 seconds +[Apr 20, 2026 - 12:57:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 537 : data loading time 0.00 +[Apr 20, 2026 - 12:57:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 537 : avg forward pass time 15.75 +[Apr 20, 2026 - 12:57:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 537 : backward pass time 12.75 +[Apr 20, 2026 - 12:57:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 537 : optimizer step time 1.06 +[Apr 20, 2026 - 12:57:31 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 537-------------------- +[Apr 20, 2026 - 12:57:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0107 iter count: 1.0 +[Apr 20, 2026 - 12:57:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[Apr 20, 2026 - 12:57:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 12:57:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 12:58:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338290 43.453697 43.207233 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.830566 70.015625 69.470703 +util 93.500000 97.000000 91.000000 +[Apr 20, 2026 - 12:58:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 538 : avg iteration time 55.49 seconds +[Apr 20, 2026 - 12:58:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 538 : data loading time 0.00 +[Apr 20, 2026 - 12:58:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 538 : avg forward pass time 15.47 +[Apr 20, 2026 - 12:58:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 538 : backward pass time 11.65 +[Apr 20, 2026 - 12:58:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 538 : optimizer step time 0.99 +[Apr 20, 2026 - 12:58:26 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 538-------------------- +[Apr 20, 2026 - 12:58:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0101 iter count: 1.0 +[Apr 20, 2026 - 12:58:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0101 iter count: 1.0 +[Apr 20, 2026 - 12:58:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 12:58:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 12:59:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338561 43.454765 43.207233 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.859863 70.152344 69.451172 +util 98.250000 99.000000 97.000000 +[Apr 20, 2026 - 12:59:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 539 : avg iteration time 50.26 seconds +[Apr 20, 2026 - 12:59:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 539 : data loading time 0.00 +[Apr 20, 2026 - 12:59:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 539 : avg forward pass time 12.50 +[Apr 20, 2026 - 12:59:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 539 : backward pass time 11.53 +[Apr 20, 2026 - 12:59:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 539 : optimizer step time 0.98 +[Apr 20, 2026 - 12:59:17 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 539-------------------- +[Apr 20, 2026 - 12:59:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0106 iter count: 1.0 +[Apr 20, 2026 - 12:59:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0106 iter count: 1.0 +[Apr 20, 2026 - 12:59:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 12:59:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 13:02:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338132 43.453049 43.207233 +peak_gpu_mem_gb 129.547819 131.549380 123.547254 +peak_gpu_mem_reserved_gb 133.619141 135.615234 127.960938 +util 70.250000 76.000000 61.000000 +[Apr 20, 2026 - 13:02:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 540 : avg iteration time 193.28 seconds +[Apr 20, 2026 - 13:02:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 540 : data loading time 0.00 +[Apr 20, 2026 - 13:02:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 540 : avg forward pass time 88.13 +[Apr 20, 2026 - 13:02:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 540 : backward pass time 8.38 +[Apr 20, 2026 - 13:02:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 540 : optimizer step time 0.36 +[Apr 20, 2026 - 13:02:30 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 540-------------------- +[Apr 20, 2026 - 13:02:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 38547.6661 iter count: 1.0 +[Apr 20, 2026 - 13:02:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6873 iter count: 1.0 +[Apr 20, 2026 - 13:02:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 38547.6661iter count: 1.0 +[Apr 20, 2026 - 13:02:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 13:02:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.1884iter count: 1.0 +[Apr 20, 2026 - 13:02:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.8799 iter count: 1.0 +[Apr 20, 2026 - 13:02:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.7483 iter count: 1.0 +[Apr 20, 2026 - 13:02:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 64091.8672iter count: 1.0 +[Apr 20, 2026 - 13:02:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 42.9384iter count: 1.0 +[Apr 20, 2026 - 13:02:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 726867.8750iter count: 1.0 +[Apr 20, 2026 - 13:02:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6873iter count: 1.0 +[Apr 20, 2026 - 13:02:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 5.4257 iter count: 1.0 +[Apr 20, 2026 - 13:02:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 51602.7305iter count: 1.0 +[Apr 20, 2026 - 13:02:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 6.6133 iter count: 1.0 +[Apr 20, 2026 - 13:02:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 554949.8750iter count: 1.0 +[Apr 20, 2026 - 13:02:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.5771 iter count: 1.0 +[Apr 20, 2026 - 13:02:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 9454.0967iter count: 1.0 +[Apr 20, 2026 - 13:02:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 3.7773 iter count: 1.0 +[Apr 20, 2026 - 13:02:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 1909.5349iter count: 1.0 +[Apr 20, 2026 - 13:02:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0123 iter count: 1.0 +[Apr 20, 2026 - 13:02:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 13:03:36 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338136 43.453049 43.207233 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.479980 72.593750 69.470703 +util 96.750000 99.000000 95.000000 +[Apr 20, 2026 - 13:03:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 541 : avg iteration time 65.90 seconds +[Apr 20, 2026 - 13:03:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 541 : data loading time 0.00 +[Apr 20, 2026 - 13:03:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 541 : avg forward pass time 20.88 +[Apr 20, 2026 - 13:03:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 541 : backward pass time 11.57 +[Apr 20, 2026 - 13:03:36 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 541 : optimizer step time 0.99 +[Apr 20, 2026 - 13:03:36 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 541-------------------- +[Apr 20, 2026 - 13:03:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0129 iter count: 1.0 +[Apr 20, 2026 - 13:03:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0129 iter count: 1.0 +[Apr 20, 2026 - 13:03:36 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 13:03:36 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 13:04:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338464 43.454292 43.207256 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.811035 69.996094 69.470703 +util 97.750000 100.000000 94.000000 +[Apr 20, 2026 - 13:04:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 542 : avg iteration time 56.39 seconds +[Apr 20, 2026 - 13:04:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 542 : data loading time 0.00 +[Apr 20, 2026 - 13:04:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 542 : avg forward pass time 15.33 +[Apr 20, 2026 - 13:04:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 542 : backward pass time 11.69 +[Apr 20, 2026 - 13:04:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 542 : optimizer step time 1.44 +[Apr 20, 2026 - 13:04:32 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 542-------------------- +[Apr 20, 2026 - 13:04:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0100 iter count: 1.0 +[Apr 20, 2026 - 13:04:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[Apr 20, 2026 - 13:04:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 13:04:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 13:05:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338589 43.454796 43.207260 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.504395 72.691406 69.509766 +util 99.500000 100.000000 99.000000 +[Apr 20, 2026 - 13:05:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 543 : avg iteration time 55.47 seconds +[Apr 20, 2026 - 13:05:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 543 : data loading time 0.00 +[Apr 20, 2026 - 13:05:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 543 : avg forward pass time 15.68 +[Apr 20, 2026 - 13:05:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 543 : backward pass time 11.54 +[Apr 20, 2026 - 13:05:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 543 : optimizer step time 1.00 +[Apr 20, 2026 - 13:05:28 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 543-------------------- +[Apr 20, 2026 - 13:05:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0149 iter count: 1.0 +[Apr 20, 2026 - 13:05:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0149 iter count: 1.0 +[Apr 20, 2026 - 13:05:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 13:05:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 13:06:19 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338582 43.454796 43.207237 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.859863 70.015625 69.470703 +util 97.750000 100.000000 92.000000 +[Apr 20, 2026 - 13:06:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 544 : avg iteration time 51.02 seconds +[Apr 20, 2026 - 13:06:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 544 : data loading time 0.00 +[Apr 20, 2026 - 13:06:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 544 : avg forward pass time 12.99 +[Apr 20, 2026 - 13:06:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 544 : backward pass time 11.56 +[Apr 20, 2026 - 13:06:19 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 544 : optimizer step time 0.98 +[Apr 20, 2026 - 13:06:19 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 544-------------------- +[Apr 20, 2026 - 13:06:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0103 iter count: 1.0 +[Apr 20, 2026 - 13:06:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[Apr 20, 2026 - 13:06:19 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 13:06:19 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 13:09:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338070 43.452709 43.207237 +peak_gpu_mem_gb 129.547819 131.549380 123.547254 +peak_gpu_mem_reserved_gb 133.516113 135.535156 127.589844 +util 65.000000 69.000000 62.000000 +[Apr 20, 2026 - 13:09:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 545 : avg iteration time 191.08 seconds +[Apr 20, 2026 - 13:09:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 545 : data loading time 0.00 +[Apr 20, 2026 - 13:09:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 545 : avg forward pass time 86.47 +[Apr 20, 2026 - 13:09:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 545 : backward pass time 8.37 +[Apr 20, 2026 - 13:09:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 545 : optimizer step time 0.34 +[Apr 20, 2026 - 13:09:30 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 545-------------------- +[Apr 20, 2026 - 13:09:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 106518.4136 iter count: 1.0 +[Apr 20, 2026 - 13:09:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5690 iter count: 1.0 +[Apr 20, 2026 - 13:09:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 106518.4136iter count: 1.0 +[Apr 20, 2026 - 13:09:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 13:09:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.3210iter count: 1.0 +[Apr 20, 2026 - 13:09:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.5522 iter count: 1.0 +[Apr 20, 2026 - 13:09:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.4928 iter count: 1.0 +[Apr 20, 2026 - 13:09:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 193307.5000iter count: 1.0 +[Apr 20, 2026 - 13:09:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 22.2974iter count: 1.0 +[Apr 20, 2026 - 13:09:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 3222291.7500iter count: 1.0 +[Apr 20, 2026 - 13:09:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5690iter count: 1.0 +[Apr 20, 2026 - 13:09:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 5.4754 iter count: 1.0 +[Apr 20, 2026 - 13:09:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 57001.9062iter count: 1.0 +[Apr 20, 2026 - 13:09:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 5.0099 iter count: 1.0 +[Apr 20, 2026 - 13:09:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 22469.1250iter count: 1.0 +[Apr 20, 2026 - 13:09:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 2.4339 iter count: 1.0 +[Apr 20, 2026 - 13:09:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 130.0439iter count: 1.0 +[Apr 20, 2026 - 13:09:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 6.8938 iter count: 1.0 +[Apr 20, 2026 - 13:09:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 972391.4375iter count: 1.0 +[Apr 20, 2026 - 13:09:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[Apr 20, 2026 - 13:09:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 13:10:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338235 43.453320 43.207253 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.791504 69.937500 69.431641 +util 99.750000 100.000000 99.000000 +[Apr 20, 2026 - 13:10:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 546 : avg iteration time 58.50 seconds +[Apr 20, 2026 - 13:10:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 546 : data loading time 0.00 +[Apr 20, 2026 - 13:10:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 546 : avg forward pass time 16.38 +[Apr 20, 2026 - 13:10:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 546 : backward pass time 12.71 +[Apr 20, 2026 - 13:10:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 546 : optimizer step time 1.47 +[Apr 20, 2026 - 13:10:28 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 546-------------------- +[Apr 20, 2026 - 13:10:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0118 iter count: 1.0 +[Apr 20, 2026 - 13:10:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0118 iter count: 1.0 +[Apr 20, 2026 - 13:10:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 13:10:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 13:11:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338431 43.454079 43.207272 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.811035 69.996094 69.451172 +util 99.500000 100.000000 99.000000 +[Apr 20, 2026 - 13:11:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 547 : avg iteration time 52.99 seconds +[Apr 20, 2026 - 13:11:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 547 : data loading time 0.00 +[Apr 20, 2026 - 13:11:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 547 : avg forward pass time 14.41 +[Apr 20, 2026 - 13:11:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 547 : backward pass time 11.55 +[Apr 20, 2026 - 13:11:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 547 : optimizer step time 1.01 +[Apr 20, 2026 - 13:11:21 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 547-------------------- +[Apr 20, 2026 - 13:11:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0104 iter count: 1.0 +[Apr 20, 2026 - 13:11:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0104 iter count: 1.0 +[Apr 20, 2026 - 13:11:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 13:11:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 13:12:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338509 43.454319 43.207317 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.830566 69.917969 69.587891 +util 98.000000 100.000000 94.000000 +[Apr 20, 2026 - 13:12:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 548 : avg iteration time 55.65 seconds +[Apr 20, 2026 - 13:12:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 548 : data loading time 0.00 +[Apr 20, 2026 - 13:12:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 548 : avg forward pass time 15.23 +[Apr 20, 2026 - 13:12:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 548 : backward pass time 11.55 +[Apr 20, 2026 - 13:12:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 548 : optimizer step time 0.97 +[Apr 20, 2026 - 13:12:17 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 548-------------------- +[Apr 20, 2026 - 13:12:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0103 iter count: 1.0 +[Apr 20, 2026 - 13:12:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[Apr 20, 2026 - 13:12:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 13:12:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 13:13:21 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338559 43.454475 43.207378 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.747559 69.898438 69.431641 +util 97.000000 99.000000 95.000000 +[Apr 20, 2026 - 13:13:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 549 : avg iteration time 64.49 seconds +[Apr 20, 2026 - 13:13:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 549 : data loading time 0.00 +[Apr 20, 2026 - 13:13:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 549 : avg forward pass time 20.18 +[Apr 20, 2026 - 13:13:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 549 : backward pass time 11.54 +[Apr 20, 2026 - 13:13:21 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 549 : optimizer step time 0.99 +[Apr 20, 2026 - 13:13:21 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 549-------------------- +[Apr 20, 2026 - 13:13:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0134 iter count: 1.0 +[Apr 20, 2026 - 13:13:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0134 iter count: 1.0 +[Apr 20, 2026 - 13:13:21 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 13:13:21 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 13:16:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338424 43.453896 43.207413 +peak_gpu_mem_gb 129.547819 131.549380 123.547254 +peak_gpu_mem_reserved_gb 133.565430 135.578125 127.667969 +util 80.250000 96.000000 72.000000 +[Apr 20, 2026 - 13:16:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 550 : avg iteration time 186.85 seconds +[Apr 20, 2026 - 13:16:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 550 : data loading time 0.00 +[Apr 20, 2026 - 13:16:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 550 : avg forward pass time 84.47 +[Apr 20, 2026 - 13:16:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 550 : backward pass time 9.25 +[Apr 20, 2026 - 13:16:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 550 : optimizer step time 0.34 +[Apr 20, 2026 - 13:16:28 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 550-------------------- +[Apr 20, 2026 - 13:16:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 25001.8013 iter count: 1.0 +[Apr 20, 2026 - 13:16:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6322 iter count: 1.0 +[Apr 20, 2026 - 13:16:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 25001.8013iter count: 1.0 +[Apr 20, 2026 - 13:16:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 13:16:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.9811iter count: 1.0 +[Apr 20, 2026 - 13:16:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.3313 iter count: 1.0 +[Apr 20, 2026 - 13:16:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.3714 iter count: 1.0 +[Apr 20, 2026 - 13:16:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 40447.3633iter count: 1.0 +[Apr 20, 2026 - 13:16:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 14.3327iter count: 1.0 +[Apr 20, 2026 - 13:16:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 342064.2812iter count: 1.0 +[Apr 20, 2026 - 13:16:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6322iter count: 1.0 +[Apr 20, 2026 - 13:16:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 6.3714 iter count: 1.0 +[Apr 20, 2026 - 13:16:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 342064.2812iter count: 1.0 +[Apr 20, 2026 - 13:16:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.8578 iter count: 1.0 +[Apr 20, 2026 - 13:16:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 16572.5293iter count: 1.0 +[Apr 20, 2026 - 13:16:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.0258 iter count: 1.0 +[Apr 20, 2026 - 13:16:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 3138.8176iter count: 1.0 +[Apr 20, 2026 - 13:16:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 6.1899 iter count: 1.0 +[Apr 20, 2026 - 13:16:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 237933.4844iter count: 1.0 +[Apr 20, 2026 - 13:16:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0096 iter count: 1.0 +[Apr 20, 2026 - 13:16:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 13:17:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338440 43.453915 43.207390 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.913574 70.015625 69.822266 +util 98.000000 100.000000 93.000000 +[Apr 20, 2026 - 13:17:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 551 : avg iteration time 63.25 seconds +[Apr 20, 2026 - 13:17:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 551 : data loading time 0.00 +[Apr 20, 2026 - 13:17:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 551 : avg forward pass time 19.55 +[Apr 20, 2026 - 13:17:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 551 : backward pass time 11.57 +[Apr 20, 2026 - 13:17:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 551 : optimizer step time 0.99 +[Apr 20, 2026 - 13:17:32 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 551-------------------- +[Apr 20, 2026 - 13:17:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0117 iter count: 1.0 +[Apr 20, 2026 - 13:17:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[Apr 20, 2026 - 13:17:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 13:17:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 13:18:37 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338419 43.453812 43.207436 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.933105 69.976562 69.880859 +util 99.500000 100.000000 98.000000 +[Apr 20, 2026 - 13:18:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 552 : avg iteration time 65.32 seconds +[Apr 20, 2026 - 13:18:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 552 : data loading time 0.00 +[Apr 20, 2026 - 13:18:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 552 : avg forward pass time 19.52 +[Apr 20, 2026 - 13:18:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 552 : backward pass time 12.38 +[Apr 20, 2026 - 13:18:37 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 552 : optimizer step time 1.59 +[Apr 20, 2026 - 13:18:37 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 552-------------------- +[Apr 20, 2026 - 13:18:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0130 iter count: 1.0 +[Apr 20, 2026 - 13:18:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0130 iter count: 1.0 +[Apr 20, 2026 - 13:18:37 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 13:18:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 13:19:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338445 43.453819 43.207420 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.864746 70.015625 69.470703 +util 99.750000 100.000000 99.000000 +[Apr 20, 2026 - 13:19:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 553 : avg iteration time 48.04 seconds +[Apr 20, 2026 - 13:19:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 553 : data loading time 0.00 +[Apr 20, 2026 - 13:19:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 553 : avg forward pass time 11.13 +[Apr 20, 2026 - 13:19:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 553 : backward pass time 13.02 +[Apr 20, 2026 - 13:19:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 553 : optimizer step time 1.21 +[Apr 20, 2026 - 13:19:25 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 553-------------------- +[Apr 20, 2026 - 13:19:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0118 iter count: 1.0 +[Apr 20, 2026 - 13:19:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0118 iter count: 1.0 +[Apr 20, 2026 - 13:19:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 13:19:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 13:20:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338473 43.453819 43.207409 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.942871 70.054688 69.820312 +util 98.000000 100.000000 93.000000 +[Apr 20, 2026 - 13:20:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 554 : avg iteration time 54.63 seconds +[Apr 20, 2026 - 13:20:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 554 : data loading time 0.00 +[Apr 20, 2026 - 13:20:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 554 : avg forward pass time 15.28 +[Apr 20, 2026 - 13:20:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 554 : backward pass time 11.57 +[Apr 20, 2026 - 13:20:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 554 : optimizer step time 0.97 +[Apr 20, 2026 - 13:20:20 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 554-------------------- +[Apr 20, 2026 - 13:20:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0098 iter count: 1.0 +[Apr 20, 2026 - 13:20:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[Apr 20, 2026 - 13:20:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 13:20:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 13:23:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338307 43.453186 43.207409 +peak_gpu_mem_gb 129.547819 131.549380 123.547254 +peak_gpu_mem_reserved_gb 133.721191 135.558594 128.253906 +util 71.500000 78.000000 64.000000 +[Apr 20, 2026 - 13:23:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 555 : avg iteration time 188.24 seconds +[Apr 20, 2026 - 13:23:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 555 : data loading time 0.00 +[Apr 20, 2026 - 13:23:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 555 : avg forward pass time 85.48 +[Apr 20, 2026 - 13:23:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 555 : backward pass time 8.35 +[Apr 20, 2026 - 13:23:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 555 : optimizer step time 0.33 +[Apr 20, 2026 - 13:23:28 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 555-------------------- +[Apr 20, 2026 - 13:23:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 46605.8657 iter count: 1.0 +[Apr 20, 2026 - 13:23:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6145 iter count: 1.0 +[Apr 20, 2026 - 13:23:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 46605.8657iter count: 1.0 +[Apr 20, 2026 - 13:23:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 13:23:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.9107iter count: 1.0 +[Apr 20, 2026 - 13:23:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.6487 iter count: 1.0 +[Apr 20, 2026 - 13:23:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.0174 iter count: 1.0 +[Apr 20, 2026 - 13:23:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 80019.3828iter count: 1.0 +[Apr 20, 2026 - 13:23:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 3.6598 iter count: 1.0 +[Apr 20, 2026 - 13:23:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 1245117.5000iter count: 1.0 +[Apr 20, 2026 - 13:23:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6145iter count: 1.0 +[Apr 20, 2026 - 13:23:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 2.8381 iter count: 1.0 +[Apr 20, 2026 - 13:23:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 291.8648iter count: 1.0 +[Apr 20, 2026 - 13:23:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.5724 iter count: 1.0 +[Apr 20, 2026 - 13:23:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 9365.9531iter count: 1.0 +[Apr 20, 2026 - 13:23:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 6.0593 iter count: 1.0 +[Apr 20, 2026 - 13:23:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 183266.7656iter count: 1.0 +[Apr 20, 2026 - 13:23:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.0641 iter count: 1.0 +[Apr 20, 2026 - 13:23:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 3388.6379iter count: 1.0 +[Apr 20, 2026 - 13:23:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0124 iter count: 1.0 +[Apr 20, 2026 - 13:23:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 13:24:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338506 43.453979 43.207409 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.494629 72.593750 69.626953 +util 97.000000 100.000000 93.000000 +[Apr 20, 2026 - 13:24:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 556 : avg iteration time 60.60 seconds +[Apr 20, 2026 - 13:24:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 556 : data loading time 0.00 +[Apr 20, 2026 - 13:24:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 556 : avg forward pass time 17.52 +[Apr 20, 2026 - 13:24:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 556 : backward pass time 11.51 +[Apr 20, 2026 - 13:24:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 556 : optimizer step time 1.01 +[Apr 20, 2026 - 13:24:28 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 556-------------------- +[Apr 20, 2026 - 13:24:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0098 iter count: 1.0 +[Apr 20, 2026 - 13:24:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[Apr 20, 2026 - 13:24:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 13:24:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 13:25:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338699 43.454739 43.207417 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.475098 72.613281 69.451172 +util 98.000000 100.000000 93.000000 +[Apr 20, 2026 - 13:25:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 557 : avg iteration time 59.16 seconds +[Apr 20, 2026 - 13:25:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 557 : data loading time 0.00 +[Apr 20, 2026 - 13:25:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 557 : avg forward pass time 17.51 +[Apr 20, 2026 - 13:25:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 557 : backward pass time 11.60 +[Apr 20, 2026 - 13:25:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 557 : optimizer step time 0.97 +[Apr 20, 2026 - 13:25:28 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 557-------------------- +[Apr 20, 2026 - 13:25:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0122 iter count: 1.0 +[Apr 20, 2026 - 13:25:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0122 iter count: 1.0 +[Apr 20, 2026 - 13:25:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 13:25:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 13:26:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338724 43.454720 43.207417 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.479980 72.750000 69.392578 +util 97.250000 99.000000 92.000000 +[Apr 20, 2026 - 13:26:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 558 : avg iteration time 50.58 seconds +[Apr 20, 2026 - 13:26:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 558 : data loading time 0.00 +[Apr 20, 2026 - 13:26:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 558 : avg forward pass time 12.88 +[Apr 20, 2026 - 13:26:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 558 : backward pass time 11.59 +[Apr 20, 2026 - 13:26:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 558 : optimizer step time 0.99 +[Apr 20, 2026 - 13:26:18 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 558-------------------- +[Apr 20, 2026 - 13:26:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0108 iter count: 1.0 +[Apr 20, 2026 - 13:26:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[Apr 20, 2026 - 13:26:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 13:26:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 13:27:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338790 43.454971 43.207424 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.567871 72.593750 69.822266 +util 99.750000 100.000000 99.000000 +[Apr 20, 2026 - 13:27:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 559 : avg iteration time 56.39 seconds +[Apr 20, 2026 - 13:27:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 559 : data loading time 0.00 +[Apr 20, 2026 - 13:27:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 559 : avg forward pass time 15.20 +[Apr 20, 2026 - 13:27:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 559 : backward pass time 11.62 +[Apr 20, 2026 - 13:27:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 559 : optimizer step time 1.40 +[Apr 20, 2026 - 13:27:14 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 559-------------------- +[Apr 20, 2026 - 13:27:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0123 iter count: 1.0 +[Apr 20, 2026 - 13:27:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0123 iter count: 1.0 +[Apr 20, 2026 - 13:27:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 13:27:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 13:30:30 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338925 43.455482 43.207455 +peak_gpu_mem_gb 129.547819 131.549380 123.547254 +peak_gpu_mem_reserved_gb 133.681641 135.613281 128.175781 +util 68.500000 75.000000 61.000000 +[Apr 20, 2026 - 13:30:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 560 : avg iteration time 195.05 seconds +[Apr 20, 2026 - 13:30:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 560 : data loading time 0.00 +[Apr 20, 2026 - 13:30:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 560 : avg forward pass time 89.06 +[Apr 20, 2026 - 13:30:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 560 : backward pass time 8.35 +[Apr 20, 2026 - 13:30:30 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 560 : optimizer step time 0.34 +[Apr 20, 2026 - 13:30:30 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 560-------------------- +[Apr 20, 2026 - 13:30:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 104670.8130 iter count: 1.0 +[Apr 20, 2026 - 13:30:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5646 iter count: 1.0 +[Apr 20, 2026 - 13:30:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 104670.8130iter count: 1.0 +[Apr 20, 2026 - 13:30:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 13:30:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.1528iter count: 1.0 +[Apr 20, 2026 - 13:30:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.6081 iter count: 1.0 +[Apr 20, 2026 - 13:30:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.4561 iter count: 1.0 +[Apr 20, 2026 - 13:30:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 143279.7344iter count: 1.0 +[Apr 20, 2026 - 13:30:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 24.9311iter count: 1.0 +[Apr 20, 2026 - 13:30:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 2994114.2500iter count: 1.0 +[Apr 20, 2026 - 13:30:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5646iter count: 1.0 +[Apr 20, 2026 - 13:30:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 2.5582 iter count: 1.0 +[Apr 20, 2026 - 13:30:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 166.7292iter count: 1.0 +[Apr 20, 2026 - 13:30:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 7.4561 iter count: 1.0 +[Apr 20, 2026 - 13:30:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 2994114.2500iter count: 1.0 +[Apr 20, 2026 - 13:30:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 2.9464 iter count: 1.0 +[Apr 20, 2026 - 13:30:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 362.4081iter count: 1.0 +[Apr 20, 2026 - 13:30:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 5.0651 iter count: 1.0 +[Apr 20, 2026 - 13:30:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 25090.3242iter count: 1.0 +[Apr 20, 2026 - 13:30:30 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0090 iter count: 1.0 +[Apr 20, 2026 - 13:30:30 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 13:31:31 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338754 43.454815 43.207458 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.933105 70.093750 69.763672 +util 98.500000 100.000000 97.000000 +[Apr 20, 2026 - 13:31:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 561 : avg iteration time 61.30 seconds +[Apr 20, 2026 - 13:31:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 561 : data loading time 0.00 +[Apr 20, 2026 - 13:31:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 561 : avg forward pass time 17.73 +[Apr 20, 2026 - 13:31:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 561 : backward pass time 13.00 +[Apr 20, 2026 - 13:31:31 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 561 : optimizer step time 1.20 +[Apr 20, 2026 - 13:31:31 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 561-------------------- +[Apr 20, 2026 - 13:31:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0134 iter count: 1.0 +[Apr 20, 2026 - 13:31:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0134 iter count: 1.0 +[Apr 20, 2026 - 13:31:31 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 13:31:31 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 13:32:29 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.339078 43.456078 43.207458 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.879395 70.132812 69.490234 +util 96.250000 99.000000 92.000000 +[Apr 20, 2026 - 13:32:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 562 : avg iteration time 57.71 seconds +[Apr 20, 2026 - 13:32:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 562 : data loading time 0.00 +[Apr 20, 2026 - 13:32:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 562 : avg forward pass time 16.74 +[Apr 20, 2026 - 13:32:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 562 : backward pass time 11.56 +[Apr 20, 2026 - 13:32:29 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 562 : optimizer step time 0.99 +[Apr 20, 2026 - 13:32:29 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 562-------------------- +[Apr 20, 2026 - 13:32:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0108 iter count: 1.0 +[Apr 20, 2026 - 13:32:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[Apr 20, 2026 - 13:32:29 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 13:32:29 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 13:33:33 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.339325 43.457062 43.207462 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.894043 70.054688 69.529297 +util 97.000000 100.000000 93.000000 +[Apr 20, 2026 - 13:33:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 563 : avg iteration time 64.81 seconds +[Apr 20, 2026 - 13:33:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 563 : data loading time 0.00 +[Apr 20, 2026 - 13:33:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 563 : avg forward pass time 20.13 +[Apr 20, 2026 - 13:33:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 563 : backward pass time 11.99 +[Apr 20, 2026 - 13:33:33 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 563 : optimizer step time 0.98 +[Apr 20, 2026 - 13:33:33 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 563-------------------- +[Apr 20, 2026 - 13:33:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0124 iter count: 1.0 +[Apr 20, 2026 - 13:33:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0124 iter count: 1.0 +[Apr 20, 2026 - 13:33:33 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 13:33:33 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 13:34:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.339355 43.457203 43.207466 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.986816 70.308594 69.820312 +util 99.250000 100.000000 99.000000 +[Apr 20, 2026 - 13:34:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 564 : avg iteration time 49.51 seconds +[Apr 20, 2026 - 13:34:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 564 : data loading time 0.00 +[Apr 20, 2026 - 13:34:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 564 : avg forward pass time 12.66 +[Apr 20, 2026 - 13:34:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 564 : backward pass time 11.57 +[Apr 20, 2026 - 13:34:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 564 : optimizer step time 0.98 +[Apr 20, 2026 - 13:34:23 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 564-------------------- +[Apr 20, 2026 - 13:34:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0102 iter count: 1.0 +[Apr 20, 2026 - 13:34:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[Apr 20, 2026 - 13:34:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 13:34:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 13:37:28 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338840 43.455086 43.207478 +peak_gpu_mem_gb 129.547819 131.549380 123.547254 +peak_gpu_mem_reserved_gb 134.092773 135.537109 129.914062 +util 73.000000 79.000000 66.000000 +[Apr 20, 2026 - 13:37:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 565 : avg iteration time 185.52 seconds +[Apr 20, 2026 - 13:37:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 565 : data loading time 0.00 +[Apr 20, 2026 - 13:37:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 565 : avg forward pass time 84.15 +[Apr 20, 2026 - 13:37:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 565 : backward pass time 8.40 +[Apr 20, 2026 - 13:37:28 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 565 : optimizer step time 0.34 +[Apr 20, 2026 - 13:37:28 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 565-------------------- +[Apr 20, 2026 - 13:37:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 150384.6816 iter count: 1.0 +[Apr 20, 2026 - 13:37:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6497 iter count: 1.0 +[Apr 20, 2026 - 13:37:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 150384.6816iter count: 1.0 +[Apr 20, 2026 - 13:37:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 13:37:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.4731iter count: 1.0 +[Apr 20, 2026 - 13:37:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.7362 iter count: 1.0 +[Apr 20, 2026 - 13:37:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.3032 iter count: 1.0 +[Apr 20, 2026 - 13:37:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 204956.6094iter count: 1.0 +[Apr 20, 2026 - 13:37:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 32.2115iter count: 1.0 +[Apr 20, 2026 - 13:37:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 2205324.5000iter count: 1.0 +[Apr 20, 2026 - 13:37:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6497iter count: 1.0 +[Apr 20, 2026 - 13:37:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.9305 iter count: 1.0 +[Apr 20, 2026 - 13:37:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 2594.1133iter count: 1.0 +[Apr 20, 2026 - 13:37:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 6.4586 iter count: 1.0 +[Apr 20, 2026 - 13:37:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 407259.8750iter count: 1.0 +[Apr 20, 2026 - 13:37:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 7.0654 iter count: 1.0 +[Apr 20, 2026 - 13:37:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 1370630.1250iter count: 1.0 +[Apr 20, 2026 - 13:37:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 5.3209 iter count: 1.0 +[Apr 20, 2026 - 13:37:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 41849.2266iter count: 1.0 +[Apr 20, 2026 - 13:37:28 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[Apr 20, 2026 - 13:37:28 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 13:38:23 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338847 43.455086 43.207478 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.597168 72.632812 69.783203 +util 99.500000 100.000000 99.000000 +[Apr 20, 2026 - 13:38:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 566 : avg iteration time 54.69 seconds +[Apr 20, 2026 - 13:38:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 566 : data loading time 0.00 +[Apr 20, 2026 - 13:38:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 566 : avg forward pass time 14.58 +[Apr 20, 2026 - 13:38:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 566 : backward pass time 11.53 +[Apr 20, 2026 - 13:38:23 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 566 : optimizer step time 0.98 +[Apr 20, 2026 - 13:38:23 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 566-------------------- +[Apr 20, 2026 - 13:38:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0103 iter count: 1.0 +[Apr 20, 2026 - 13:38:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[Apr 20, 2026 - 13:38:23 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 13:38:23 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 13:39:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.339128 43.456200 43.207497 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.869629 70.035156 69.451172 +util 97.500000 100.000000 92.000000 +[Apr 20, 2026 - 13:39:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 567 : avg iteration time 57.02 seconds +[Apr 20, 2026 - 13:39:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 567 : data loading time 0.00 +[Apr 20, 2026 - 13:39:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 567 : avg forward pass time 15.70 +[Apr 20, 2026 - 13:39:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 567 : backward pass time 12.92 +[Apr 20, 2026 - 13:39:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 567 : optimizer step time 1.11 +[Apr 20, 2026 - 13:39:20 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 567-------------------- +[Apr 20, 2026 - 13:39:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0112 iter count: 1.0 +[Apr 20, 2026 - 13:39:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[Apr 20, 2026 - 13:39:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 13:39:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 13:40:20 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.339128 43.456192 43.207500 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.582520 72.906250 69.451172 +util 96.250000 99.000000 93.000000 +[Apr 20, 2026 - 13:40:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 568 : avg iteration time 60.11 seconds +[Apr 20, 2026 - 13:40:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 568 : data loading time 0.00 +[Apr 20, 2026 - 13:40:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 568 : avg forward pass time 18.00 +[Apr 20, 2026 - 13:40:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 568 : backward pass time 11.54 +[Apr 20, 2026 - 13:40:20 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 568 : optimizer step time 0.98 +[Apr 20, 2026 - 13:40:20 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 568-------------------- +[Apr 20, 2026 - 13:40:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0126 iter count: 1.0 +[Apr 20, 2026 - 13:40:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0126 iter count: 1.0 +[Apr 20, 2026 - 13:40:20 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 13:40:20 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 13:41:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.339140 43.456200 43.207500 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.616699 72.925781 69.548828 +util 99.250000 100.000000 99.000000 +[Apr 20, 2026 - 13:41:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 569 : avg iteration time 64.48 seconds +[Apr 20, 2026 - 13:41:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 569 : data loading time 0.00 +[Apr 20, 2026 - 13:41:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 569 : avg forward pass time 20.12 +[Apr 20, 2026 - 13:41:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 569 : backward pass time 11.58 +[Apr 20, 2026 - 13:41:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 569 : optimizer step time 0.98 +[Apr 20, 2026 - 13:41:25 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 569-------------------- +[Apr 20, 2026 - 13:41:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0119 iter count: 1.0 +[Apr 20, 2026 - 13:41:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[Apr 20, 2026 - 13:41:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 13:41:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 13:44:26 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338960 43.455460 43.207516 +peak_gpu_mem_gb 129.547819 131.549380 123.547254 +peak_gpu_mem_reserved_gb 133.608887 135.537109 127.843750 +util 80.250000 95.000000 67.000000 +[Apr 20, 2026 - 13:44:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 570 : avg iteration time 181.46 seconds +[Apr 20, 2026 - 13:44:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 570 : data loading time 0.00 +[Apr 20, 2026 - 13:44:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 570 : avg forward pass time 81.94 +[Apr 20, 2026 - 13:44:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 570 : backward pass time 8.97 +[Apr 20, 2026 - 13:44:26 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 570 : optimizer step time 0.34 +[Apr 20, 2026 - 13:44:26 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 570-------------------- +[Apr 20, 2026 - 13:44:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 548173.4294 iter count: 1.0 +[Apr 20, 2026 - 13:44:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5974 iter count: 1.0 +[Apr 20, 2026 - 13:44:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 548173.4294iter count: 1.0 +[Apr 20, 2026 - 13:44:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 13:44:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.3347iter count: 1.0 +[Apr 20, 2026 - 13:44:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.4109 iter count: 1.0 +[Apr 20, 2026 - 13:44:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 8.2535 iter count: 1.0 +[Apr 20, 2026 - 13:44:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 674212.9375iter count: 1.0 +[Apr 20, 2026 - 13:44:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 2.2748 iter count: 1.0 +[Apr 20, 2026 - 13:44:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 14754759.0000iter count: 1.0 +[Apr 20, 2026 - 13:44:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5974iter count: 1.0 +[Apr 20, 2026 - 13:44:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.0875 iter count: 1.0 +[Apr 20, 2026 - 13:44:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 3551.0117iter count: 1.0 +[Apr 20, 2026 - 13:44:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 6.0270 iter count: 1.0 +[Apr 20, 2026 - 13:44:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 171768.6094iter count: 1.0 +[Apr 20, 2026 - 13:44:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 2.0212 iter count: 1.0 +[Apr 20, 2026 - 13:44:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 56.9682 iter count: 1.0 +[Apr 20, 2026 - 13:44:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 3.1108 iter count: 1.0 +[Apr 20, 2026 - 13:44:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 503.4916iter count: 1.0 +[Apr 20, 2026 - 13:44:26 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[Apr 20, 2026 - 13:44:26 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 13:45:22 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338983 43.455536 43.207531 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.894043 70.425781 69.490234 +util 96.500000 100.000000 92.000000 +[Apr 20, 2026 - 13:45:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 571 : avg iteration time 55.40 seconds +[Apr 20, 2026 - 13:45:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 571 : data loading time 0.00 +[Apr 20, 2026 - 13:45:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 571 : avg forward pass time 15.64 +[Apr 20, 2026 - 13:45:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 571 : backward pass time 11.53 +[Apr 20, 2026 - 13:45:22 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 571 : optimizer step time 1.01 +[Apr 20, 2026 - 13:45:22 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 571-------------------- +[Apr 20, 2026 - 13:45:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0126 iter count: 1.0 +[Apr 20, 2026 - 13:45:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0126 iter count: 1.0 +[Apr 20, 2026 - 13:45:22 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 13:45:22 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 13:46:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.339042 43.455772 43.207531 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.665527 72.361328 69.937500 +util 96.750000 99.000000 93.000000 +[Apr 20, 2026 - 13:46:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 572 : avg iteration time 56.20 seconds +[Apr 20, 2026 - 13:46:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 572 : data loading time 0.00 +[Apr 20, 2026 - 13:46:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 572 : avg forward pass time 15.28 +[Apr 20, 2026 - 13:46:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 572 : backward pass time 11.61 +[Apr 20, 2026 - 13:46:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 572 : optimizer step time 0.98 +[Apr 20, 2026 - 13:46:18 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 572-------------------- +[Apr 20, 2026 - 13:46:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0115 iter count: 1.0 +[Apr 20, 2026 - 13:46:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[Apr 20, 2026 - 13:46:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 13:46:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 13:47:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.339038 43.455750 43.207531 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.494629 72.361328 69.761719 +util 97.000000 100.000000 93.000000 +[Apr 20, 2026 - 13:47:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 573 : avg iteration time 59.60 seconds +[Apr 20, 2026 - 13:47:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 573 : data loading time 0.00 +[Apr 20, 2026 - 13:47:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 573 : avg forward pass time 17.25 +[Apr 20, 2026 - 13:47:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 573 : backward pass time 12.51 +[Apr 20, 2026 - 13:47:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 573 : optimizer step time 1.01 +[Apr 20, 2026 - 13:47:17 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 573-------------------- +[Apr 20, 2026 - 13:47:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0112 iter count: 1.0 +[Apr 20, 2026 - 13:47:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[Apr 20, 2026 - 13:47:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 13:47:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 13:48:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.339038 43.455750 43.207531 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.913574 70.035156 69.781250 +util 97.500000 100.000000 92.000000 +[Apr 20, 2026 - 13:48:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 574 : avg iteration time 49.36 seconds +[Apr 20, 2026 - 13:48:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 574 : data loading time 0.00 +[Apr 20, 2026 - 13:48:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 574 : avg forward pass time 12.61 +[Apr 20, 2026 - 13:48:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 574 : backward pass time 11.55 +[Apr 20, 2026 - 13:48:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 574 : optimizer step time 0.99 +[Apr 20, 2026 - 13:48:07 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 574-------------------- +[Apr 20, 2026 - 13:48:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0121 iter count: 1.0 +[Apr 20, 2026 - 13:48:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[Apr 20, 2026 - 13:48:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 13:48:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 13:51:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338758 43.454552 43.207611 +peak_gpu_mem_gb 129.547819 131.549380 123.547254 +peak_gpu_mem_reserved_gb 133.984863 135.593750 129.250000 +util 65.750000 79.000000 53.000000 +[Apr 20, 2026 - 13:51:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 575 : avg iteration time 190.43 seconds +[Apr 20, 2026 - 13:51:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 575 : data loading time 0.00 +[Apr 20, 2026 - 13:51:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 575 : avg forward pass time 86.45 +[Apr 20, 2026 - 13:51:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 575 : backward pass time 8.85 +[Apr 20, 2026 - 13:51:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 575 : optimizer step time 0.33 +[Apr 20, 2026 - 13:51:17 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 575-------------------- +[Apr 20, 2026 - 13:51:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 39649.0861 iter count: 1.0 +[Apr 20, 2026 - 13:51:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6943 iter count: 1.0 +[Apr 20, 2026 - 13:51:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 39649.0861iter count: 1.0 +[Apr 20, 2026 - 13:51:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 13:51:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.4389iter count: 1.0 +[Apr 20, 2026 - 13:51:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.8554 iter count: 1.0 +[Apr 20, 2026 - 13:51:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.5604 iter count: 1.0 +[Apr 20, 2026 - 13:51:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 56138.6836iter count: 1.0 +[Apr 20, 2026 - 13:51:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 40.8864iter count: 1.0 +[Apr 20, 2026 - 13:51:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 499228.4375iter count: 1.0 +[Apr 20, 2026 - 13:51:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6943iter count: 1.0 +[Apr 20, 2026 - 13:51:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.1905 iter count: 1.0 +[Apr 20, 2026 - 13:51:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 590.4597iter count: 1.0 +[Apr 20, 2026 - 13:51:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 6.0082 iter count: 1.0 +[Apr 20, 2026 - 13:51:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 165445.2500iter count: 1.0 +[Apr 20, 2026 - 13:51:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 5.6883 iter count: 1.0 +[Apr 20, 2026 - 13:51:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 87263.0391iter count: 1.0 +[Apr 20, 2026 - 13:51:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 3.9480 iter count: 1.0 +[Apr 20, 2026 - 13:51:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 2686.4634iter count: 1.0 +[Apr 20, 2026 - 13:51:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[Apr 20, 2026 - 13:51:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 13:52:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338812 43.454762 43.207611 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.889160 70.015625 69.763672 +util 97.250000 100.000000 91.000000 +[Apr 20, 2026 - 13:52:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 576 : avg iteration time 54.75 seconds +[Apr 20, 2026 - 13:52:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 576 : data loading time 0.00 +[Apr 20, 2026 - 13:52:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 576 : avg forward pass time 14.74 +[Apr 20, 2026 - 13:52:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 576 : backward pass time 12.68 +[Apr 20, 2026 - 13:52:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 576 : optimizer step time 1.05 +[Apr 20, 2026 - 13:52:12 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 576-------------------- +[Apr 20, 2026 - 13:52:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0098 iter count: 1.0 +[Apr 20, 2026 - 13:52:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[Apr 20, 2026 - 13:52:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 13:52:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 13:53:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.339111 43.455959 43.207611 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.869629 69.937500 69.783203 +util 99.500000 100.000000 99.000000 +[Apr 20, 2026 - 13:53:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 577 : avg iteration time 60.97 seconds +[Apr 20, 2026 - 13:53:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 577 : data loading time 0.00 +[Apr 20, 2026 - 13:53:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 577 : avg forward pass time 18.18 +[Apr 20, 2026 - 13:53:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 577 : backward pass time 11.58 +[Apr 20, 2026 - 13:53:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 577 : optimizer step time 0.98 +[Apr 20, 2026 - 13:53:13 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 577-------------------- +[Apr 20, 2026 - 13:53:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0100 iter count: 1.0 +[Apr 20, 2026 - 13:53:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[Apr 20, 2026 - 13:53:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 13:53:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 13:54:09 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.339173 43.456226 43.207611 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.913574 69.996094 69.822266 +util 93.000000 97.000000 90.000000 +[Apr 20, 2026 - 13:54:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 578 : avg iteration time 56.49 seconds +[Apr 20, 2026 - 13:54:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 578 : data loading time 0.00 +[Apr 20, 2026 - 13:54:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 578 : avg forward pass time 15.70 +[Apr 20, 2026 - 13:54:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 578 : backward pass time 12.53 +[Apr 20, 2026 - 13:54:09 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 578 : optimizer step time 1.00 +[Apr 20, 2026 - 13:54:09 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 578-------------------- +[Apr 20, 2026 - 13:54:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0107 iter count: 1.0 +[Apr 20, 2026 - 13:54:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[Apr 20, 2026 - 13:54:09 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 13:54:09 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 13:55:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.339214 43.456203 43.207661 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.401855 72.359375 69.392578 +util 98.250000 100.000000 97.000000 +[Apr 20, 2026 - 13:55:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 579 : avg iteration time 63.92 seconds +[Apr 20, 2026 - 13:55:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 579 : data loading time 0.00 +[Apr 20, 2026 - 13:55:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 579 : avg forward pass time 19.56 +[Apr 20, 2026 - 13:55:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 579 : backward pass time 11.58 +[Apr 20, 2026 - 13:55:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 579 : optimizer step time 1.05 +[Apr 20, 2026 - 13:55:13 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 579-------------------- +[Apr 20, 2026 - 13:55:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0102 iter count: 1.0 +[Apr 20, 2026 - 13:55:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0102 iter count: 1.0 +[Apr 20, 2026 - 13:55:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 13:55:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 13:58:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.339210 43.455673 43.207745 +peak_gpu_mem_gb 129.547819 131.549380 123.547254 +peak_gpu_mem_reserved_gb 133.564453 135.654297 127.626953 +util 70.750000 77.000000 63.000000 +[Apr 20, 2026 - 13:58:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 580 : avg iteration time 185.05 seconds +[Apr 20, 2026 - 13:58:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 580 : data loading time 0.00 +[Apr 20, 2026 - 13:58:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 580 : avg forward pass time 84.06 +[Apr 20, 2026 - 13:58:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 580 : backward pass time 8.34 +[Apr 20, 2026 - 13:58:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 580 : optimizer step time 0.33 +[Apr 20, 2026 - 13:58:18 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 580-------------------- +[Apr 20, 2026 - 13:58:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 282783.5443 iter count: 1.0 +[Apr 20, 2026 - 13:58:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5701 iter count: 1.0 +[Apr 20, 2026 - 13:58:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 282783.5443iter count: 1.0 +[Apr 20, 2026 - 13:58:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 13:58:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.2106iter count: 1.0 +[Apr 20, 2026 - 13:58:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.4536 iter count: 1.0 +[Apr 20, 2026 - 13:58:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 8.0145 iter count: 1.0 +[Apr 20, 2026 - 13:58:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 330108.4062iter count: 1.0 +[Apr 20, 2026 - 13:58:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 18.3048iter count: 1.0 +[Apr 20, 2026 - 13:58:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 9148134.0000iter count: 1.0 +[Apr 20, 2026 - 13:58:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5701iter count: 1.0 +[Apr 20, 2026 - 13:58:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.6953 iter count: 1.0 +[Apr 20, 2026 - 13:58:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 1620.7498iter count: 1.0 +[Apr 20, 2026 - 13:58:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 6.8461 iter count: 1.0 +[Apr 20, 2026 - 13:58:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 883988.8125iter count: 1.0 +[Apr 20, 2026 - 13:58:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.4821 iter count: 1.0 +[Apr 20, 2026 - 13:58:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 7817.5430iter count: 1.0 +[Apr 20, 2026 - 13:58:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 5.1139 iter count: 1.0 +[Apr 20, 2026 - 13:58:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 27663.9180iter count: 1.0 +[Apr 20, 2026 - 13:58:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[Apr 20, 2026 - 13:58:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 13:59:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.339211 43.455673 43.207745 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.460449 72.535156 69.392578 +util 97.750000 100.000000 93.000000 +[Apr 20, 2026 - 13:59:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 581 : avg iteration time 57.98 seconds +[Apr 20, 2026 - 13:59:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 581 : data loading time 0.00 +[Apr 20, 2026 - 13:59:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 581 : avg forward pass time 16.41 +[Apr 20, 2026 - 13:59:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 581 : backward pass time 12.59 +[Apr 20, 2026 - 13:59:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 581 : optimizer step time 1.02 +[Apr 20, 2026 - 13:59:16 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 581-------------------- +[Apr 20, 2026 - 13:59:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0118 iter count: 1.0 +[Apr 20, 2026 - 13:59:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0118 iter count: 1.0 +[Apr 20, 2026 - 13:59:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 13:59:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 14:00:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.339211 43.455677 43.207745 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.543457 72.554688 69.646484 +util 95.000000 99.000000 92.000000 +[Apr 20, 2026 - 14:00:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 582 : avg iteration time 52.07 seconds +[Apr 20, 2026 - 14:00:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 582 : data loading time 0.00 +[Apr 20, 2026 - 14:00:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 582 : avg forward pass time 13.96 +[Apr 20, 2026 - 14:00:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 582 : backward pass time 11.53 +[Apr 20, 2026 - 14:00:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 582 : optimizer step time 1.00 +[Apr 20, 2026 - 14:00:08 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 582-------------------- +[Apr 20, 2026 - 14:00:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0114 iter count: 1.0 +[Apr 20, 2026 - 14:00:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0114 iter count: 1.0 +[Apr 20, 2026 - 14:00:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 14:00:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 14:00:56 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.339226 43.455677 43.207760 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.489746 72.574219 69.685547 +util 98.000000 100.000000 95.000000 +[Apr 20, 2026 - 14:00:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 583 : avg iteration time 47.42 seconds +[Apr 20, 2026 - 14:00:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 583 : data loading time 0.00 +[Apr 20, 2026 - 14:00:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 583 : avg forward pass time 11.27 +[Apr 20, 2026 - 14:00:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 583 : backward pass time 11.54 +[Apr 20, 2026 - 14:00:56 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 583 : optimizer step time 1.15 +[Apr 20, 2026 - 14:00:56 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 583-------------------- +[Apr 20, 2026 - 14:00:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0138 iter count: 1.0 +[Apr 20, 2026 - 14:00:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0138 iter count: 1.0 +[Apr 20, 2026 - 14:00:56 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 14:00:56 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 14:01:57 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.339218 43.455688 43.207745 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.572754 72.671875 69.626953 +util 97.000000 100.000000 93.000000 +[Apr 20, 2026 - 14:01:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 584 : avg iteration time 60.74 seconds +[Apr 20, 2026 - 14:01:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 584 : data loading time 0.00 +[Apr 20, 2026 - 14:01:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 584 : avg forward pass time 18.34 +[Apr 20, 2026 - 14:01:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 584 : backward pass time 11.52 +[Apr 20, 2026 - 14:01:57 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 584 : optimizer step time 1.00 +[Apr 20, 2026 - 14:01:57 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 584-------------------- +[Apr 20, 2026 - 14:01:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0111 iter count: 1.0 +[Apr 20, 2026 - 14:01:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0111 iter count: 1.0 +[Apr 20, 2026 - 14:01:57 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 14:01:57 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 14:05:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338926 43.454475 43.207745 +peak_gpu_mem_gb 129.547819 131.549380 123.547254 +peak_gpu_mem_reserved_gb 133.608398 135.673828 127.824219 +util 72.250000 76.000000 69.000000 +[Apr 20, 2026 - 14:05:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 585 : avg iteration time 193.52 seconds +[Apr 20, 2026 - 14:05:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 585 : data loading time 0.00 +[Apr 20, 2026 - 14:05:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 585 : avg forward pass time 88.24 +[Apr 20, 2026 - 14:05:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 585 : backward pass time 8.34 +[Apr 20, 2026 - 14:05:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 585 : optimizer step time 0.34 +[Apr 20, 2026 - 14:05:10 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 585-------------------- +[Apr 20, 2026 - 14:05:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 14187.6428 iter count: 1.0 +[Apr 20, 2026 - 14:05:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6478 iter count: 1.0 +[Apr 20, 2026 - 14:05:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 14187.6428iter count: 1.0 +[Apr 20, 2026 - 14:05:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 14:05:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.9412iter count: 1.0 +[Apr 20, 2026 - 14:05:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.4040 iter count: 1.0 +[Apr 20, 2026 - 14:05:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.1730 iter count: 1.0 +[Apr 20, 2026 - 14:05:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 26858.8320iter count: 1.0 +[Apr 20, 2026 - 14:05:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 16.5757iter count: 1.0 +[Apr 20, 2026 - 14:05:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 230022.4062iter count: 1.0 +[Apr 20, 2026 - 14:05:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6478iter count: 1.0 +[Apr 20, 2026 - 14:05:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.3997 iter count: 1.0 +[Apr 20, 2026 - 14:05:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 6630.5073iter count: 1.0 +[Apr 20, 2026 - 14:05:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.9244 iter count: 1.0 +[Apr 20, 2026 - 14:05:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 18934.6934iter count: 1.0 +[Apr 20, 2026 - 14:05:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 5.6721 iter count: 1.0 +[Apr 20, 2026 - 14:05:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 84477.1562iter count: 1.0 +[Apr 20, 2026 - 14:05:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.5767 iter count: 1.0 +[Apr 20, 2026 - 14:05:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 9446.9043iter count: 1.0 +[Apr 20, 2026 - 14:05:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0128 iter count: 1.0 +[Apr 20, 2026 - 14:05:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 14:06:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338990 43.454723 43.207798 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.621582 72.632812 69.861328 +util 97.500000 100.000000 91.000000 +[Apr 20, 2026 - 14:06:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 586 : avg iteration time 65.62 seconds +[Apr 20, 2026 - 14:06:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 586 : data loading time 0.00 +[Apr 20, 2026 - 14:06:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 586 : avg forward pass time 20.75 +[Apr 20, 2026 - 14:06:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 586 : backward pass time 11.57 +[Apr 20, 2026 - 14:06:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 586 : optimizer step time 0.99 +[Apr 20, 2026 - 14:06:16 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 586-------------------- +[Apr 20, 2026 - 14:06:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0119 iter count: 1.0 +[Apr 20, 2026 - 14:06:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[Apr 20, 2026 - 14:06:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 14:06:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 14:07:15 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.339264 43.455864 43.207752 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.626465 72.691406 69.861328 +util 97.750000 100.000000 95.000000 +[Apr 20, 2026 - 14:07:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 587 : avg iteration time 59.30 seconds +[Apr 20, 2026 - 14:07:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 587 : data loading time 0.00 +[Apr 20, 2026 - 14:07:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 587 : avg forward pass time 16.56 +[Apr 20, 2026 - 14:07:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 587 : backward pass time 12.00 +[Apr 20, 2026 - 14:07:15 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 587 : optimizer step time 1.55 +[Apr 20, 2026 - 14:07:15 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 587-------------------- +[Apr 20, 2026 - 14:07:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0127 iter count: 1.0 +[Apr 20, 2026 - 14:07:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0127 iter count: 1.0 +[Apr 20, 2026 - 14:07:15 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 14:07:15 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 14:08:10 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.339284 43.455875 43.207798 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.543457 72.769531 69.451172 +util 97.250000 99.000000 95.000000 +[Apr 20, 2026 - 14:08:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 588 : avg iteration time 55.22 seconds +[Apr 20, 2026 - 14:08:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 588 : data loading time 0.00 +[Apr 20, 2026 - 14:08:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 588 : avg forward pass time 15.57 +[Apr 20, 2026 - 14:08:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 588 : backward pass time 11.54 +[Apr 20, 2026 - 14:08:10 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 588 : optimizer step time 1.00 +[Apr 20, 2026 - 14:08:10 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 588-------------------- +[Apr 20, 2026 - 14:08:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0119 iter count: 1.0 +[Apr 20, 2026 - 14:08:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0119 iter count: 1.0 +[Apr 20, 2026 - 14:08:10 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 14:08:10 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 14:09:06 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.339388 43.456352 43.207752 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.577637 72.769531 69.568359 +util 97.250000 100.000000 91.000000 +[Apr 20, 2026 - 14:09:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 589 : avg iteration time 55.68 seconds +[Apr 20, 2026 - 14:09:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 589 : data loading time 0.00 +[Apr 20, 2026 - 14:09:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 589 : avg forward pass time 15.21 +[Apr 20, 2026 - 14:09:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 589 : backward pass time 11.53 +[Apr 20, 2026 - 14:09:06 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 589 : optimizer step time 1.06 +[Apr 20, 2026 - 14:09:06 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 589-------------------- +[Apr 20, 2026 - 14:09:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0086 iter count: 1.0 +[Apr 20, 2026 - 14:09:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0086 iter count: 1.0 +[Apr 20, 2026 - 14:09:06 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 14:09:06 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 14:12:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.339291 43.455925 43.207767 +peak_gpu_mem_gb 129.547819 131.549380 123.547254 +peak_gpu_mem_reserved_gb 133.697266 135.576172 128.195312 +util 80.250000 95.000000 67.000000 +[Apr 20, 2026 - 14:12:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 590 : avg iteration time 191.23 seconds +[Apr 20, 2026 - 14:12:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 590 : data loading time 0.00 +[Apr 20, 2026 - 14:12:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 590 : avg forward pass time 86.76 +[Apr 20, 2026 - 14:12:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 590 : backward pass time 9.05 +[Apr 20, 2026 - 14:12:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 590 : optimizer step time 0.33 +[Apr 20, 2026 - 14:12:17 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 590-------------------- +[Apr 20, 2026 - 14:12:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 6860.1318 iter count: 1.0 +[Apr 20, 2026 - 14:12:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6177 iter count: 1.0 +[Apr 20, 2026 - 14:12:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 6860.1318iter count: 1.0 +[Apr 20, 2026 - 14:12:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 14:12:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.9479iter count: 1.0 +[Apr 20, 2026 - 14:12:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.8408 iter count: 1.0 +[Apr 20, 2026 - 14:12:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 5.6059 iter count: 1.0 +[Apr 20, 2026 - 14:12:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 10501.7666iter count: 1.0 +[Apr 20, 2026 - 14:12:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 39.7097iter count: 1.0 +[Apr 20, 2026 - 14:12:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 73995.3516iter count: 1.0 +[Apr 20, 2026 - 14:12:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6177iter count: 1.0 +[Apr 20, 2026 - 14:12:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.2011 iter count: 1.0 +[Apr 20, 2026 - 14:12:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 603.1342iter count: 1.0 +[Apr 20, 2026 - 14:12:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 5.3160 iter count: 1.0 +[Apr 20, 2026 - 14:12:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 41436.7617iter count: 1.0 +[Apr 20, 2026 - 14:12:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 3.7835 iter count: 1.0 +[Apr 20, 2026 - 14:12:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 1933.3341iter count: 1.0 +[Apr 20, 2026 - 14:12:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.1025 iter count: 1.0 +[Apr 20, 2026 - 14:12:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 3659.3403iter count: 1.0 +[Apr 20, 2026 - 14:12:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0125 iter count: 1.0 +[Apr 20, 2026 - 14:12:17 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 14:13:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.339345 43.456150 43.207783 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.811035 69.917969 69.585938 +util 98.000000 100.000000 93.000000 +[Apr 20, 2026 - 14:13:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 591 : avg iteration time 57.18 seconds +[Apr 20, 2026 - 14:13:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 591 : data loading time 0.00 +[Apr 20, 2026 - 14:13:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 591 : avg forward pass time 16.53 +[Apr 20, 2026 - 14:13:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 591 : backward pass time 11.54 +[Apr 20, 2026 - 14:13:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 591 : optimizer step time 0.99 +[Apr 20, 2026 - 14:13:14 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 591-------------------- +[Apr 20, 2026 - 14:13:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0118 iter count: 1.0 +[Apr 20, 2026 - 14:13:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0118 iter count: 1.0 +[Apr 20, 2026 - 14:13:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 14:13:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 14:14:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.339362 43.456150 43.207809 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.801270 69.937500 69.468750 +util 96.750000 99.000000 93.000000 +[Apr 20, 2026 - 14:14:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 592 : avg iteration time 59.07 seconds +[Apr 20, 2026 - 14:14:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 592 : data loading time 0.00 +[Apr 20, 2026 - 14:14:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 592 : avg forward pass time 16.62 +[Apr 20, 2026 - 14:14:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 592 : backward pass time 12.03 +[Apr 20, 2026 - 14:14:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 592 : optimizer step time 1.56 +[Apr 20, 2026 - 14:14:13 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 592-------------------- +[Apr 20, 2026 - 14:14:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0105 iter count: 1.0 +[Apr 20, 2026 - 14:14:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[Apr 20, 2026 - 14:14:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 14:14:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 14:15:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.339375 43.456161 43.207840 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.757324 69.917969 69.566406 +util 98.750000 100.000000 97.000000 +[Apr 20, 2026 - 14:15:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 593 : avg iteration time 62.85 seconds +[Apr 20, 2026 - 14:15:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 593 : data loading time 0.00 +[Apr 20, 2026 - 14:15:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 593 : avg forward pass time 19.38 +[Apr 20, 2026 - 14:15:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 593 : backward pass time 11.54 +[Apr 20, 2026 - 14:15:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 593 : optimizer step time 0.99 +[Apr 20, 2026 - 14:15:16 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 593-------------------- +[Apr 20, 2026 - 14:15:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0122 iter count: 1.0 +[Apr 20, 2026 - 14:15:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0122 iter count: 1.0 +[Apr 20, 2026 - 14:15:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 14:15:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 14:16:08 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.339366 43.456150 43.207798 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.679199 69.880859 69.449219 +util 99.750000 100.000000 99.000000 +[Apr 20, 2026 - 14:16:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 594 : avg iteration time 51.47 seconds +[Apr 20, 2026 - 14:16:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 594 : data loading time 0.00 +[Apr 20, 2026 - 14:16:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 594 : avg forward pass time 12.99 +[Apr 20, 2026 - 14:16:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 594 : backward pass time 12.67 +[Apr 20, 2026 - 14:16:08 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 594 : optimizer step time 1.28 +[Apr 20, 2026 - 14:16:08 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 594-------------------- +[Apr 20, 2026 - 14:16:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0107 iter count: 1.0 +[Apr 20, 2026 - 14:16:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0107 iter count: 1.0 +[Apr 20, 2026 - 14:16:08 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 14:16:08 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 14:19:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.339029 43.454815 43.207794 +peak_gpu_mem_gb 129.547819 131.549380 123.547254 +peak_gpu_mem_reserved_gb 133.516113 135.539062 127.509766 +util 71.750000 75.000000 69.000000 +[Apr 20, 2026 - 14:19:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 595 : avg iteration time 184.95 seconds +[Apr 20, 2026 - 14:19:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 595 : data loading time 0.00 +[Apr 20, 2026 - 14:19:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 595 : avg forward pass time 83.98 +[Apr 20, 2026 - 14:19:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 595 : backward pass time 8.34 +[Apr 20, 2026 - 14:19:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 595 : optimizer step time 0.34 +[Apr 20, 2026 - 14:19:13 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 595-------------------- +[Apr 20, 2026 - 14:19:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 846339.8403 iter count: 1.0 +[Apr 20, 2026 - 14:19:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.7126 iter count: 1.0 +[Apr 20, 2026 - 14:19:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 846339.8403iter count: 1.0 +[Apr 20, 2026 - 14:19:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 14:19:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.9632iter count: 1.0 +[Apr 20, 2026 - 14:19:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 2.3545 iter count: 1.0 +[Apr 20, 2026 - 14:19:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 8.5297 iter count: 1.0 +[Apr 20, 2026 - 14:19:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 1029765.9375iter count: 1.0 +[Apr 20, 2026 - 14:19:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 110.9342iter count: 1.0 +[Apr 20, 2026 - 14:19:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 25632900.0000iter count: 1.0 +[Apr 20, 2026 - 14:19:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.7126iter count: 1.0 +[Apr 20, 2026 - 14:19:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.2358 iter count: 1.0 +[Apr 20, 2026 - 14:19:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 646.5344iter count: 1.0 +[Apr 20, 2026 - 14:19:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 6.4864 iter count: 1.0 +[Apr 20, 2026 - 14:19:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 430567.0938iter count: 1.0 +[Apr 20, 2026 - 14:19:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.6375 iter count: 1.0 +[Apr 20, 2026 - 14:19:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 10667.0391iter count: 1.0 +[Apr 20, 2026 - 14:19:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 5.5563 iter count: 1.0 +[Apr 20, 2026 - 14:19:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 67008.4219iter count: 1.0 +[Apr 20, 2026 - 14:19:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0130 iter count: 1.0 +[Apr 20, 2026 - 14:19:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 14:20:13 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.339091 43.455063 43.207794 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.786621 69.976562 69.546875 +util 96.250000 100.000000 91.000000 +[Apr 20, 2026 - 14:20:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 596 : avg iteration time 60.28 seconds +[Apr 20, 2026 - 14:20:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 596 : data loading time 0.00 +[Apr 20, 2026 - 14:20:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 596 : avg forward pass time 18.03 +[Apr 20, 2026 - 14:20:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 596 : backward pass time 11.67 +[Apr 20, 2026 - 14:20:13 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 596 : optimizer step time 0.97 +[Apr 20, 2026 - 14:20:13 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 596-------------------- +[Apr 20, 2026 - 14:20:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0131 iter count: 1.0 +[Apr 20, 2026 - 14:20:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0131 iter count: 1.0 +[Apr 20, 2026 - 14:20:13 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 14:20:13 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 14:21:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.339375 43.456200 43.207794 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.528809 72.398438 69.839844 +util 96.750000 99.000000 90.000000 +[Apr 20, 2026 - 14:21:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 597 : avg iteration time 52.56 seconds +[Apr 20, 2026 - 14:21:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 597 : data loading time 0.00 +[Apr 20, 2026 - 14:21:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 597 : avg forward pass time 14.13 +[Apr 20, 2026 - 14:21:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 597 : backward pass time 11.57 +[Apr 20, 2026 - 14:21:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 597 : optimizer step time 0.98 +[Apr 20, 2026 - 14:21:05 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 597-------------------- +[Apr 20, 2026 - 14:21:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0115 iter count: 1.0 +[Apr 20, 2026 - 14:21:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0115 iter count: 1.0 +[Apr 20, 2026 - 14:21:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 14:21:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 14:22:07 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.339383 43.456203 43.207794 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.874512 69.957031 69.820312 +util 97.500000 100.000000 92.000000 +[Apr 20, 2026 - 14:22:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 598 : avg iteration time 61.42 seconds +[Apr 20, 2026 - 14:22:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 598 : data loading time 0.00 +[Apr 20, 2026 - 14:22:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 598 : avg forward pass time 18.65 +[Apr 20, 2026 - 14:22:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 598 : backward pass time 11.55 +[Apr 20, 2026 - 14:22:07 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 598 : optimizer step time 1.01 +[Apr 20, 2026 - 14:22:07 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 598-------------------- +[Apr 20, 2026 - 14:22:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0103 iter count: 1.0 +[Apr 20, 2026 - 14:22:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0103 iter count: 1.0 +[Apr 20, 2026 - 14:22:07 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 14:22:07 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 14:23:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.339310 43.455948 43.207794 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.830566 69.996094 69.546875 +util 99.250000 100.000000 98.000000 +[Apr 20, 2026 - 14:23:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 599 : avg iteration time 54.35 seconds +[Apr 20, 2026 - 14:23:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 599 : data loading time 0.00 +[Apr 20, 2026 - 14:23:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 599 : avg forward pass time 14.74 +[Apr 20, 2026 - 14:23:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 599 : backward pass time 11.54 +[Apr 20, 2026 - 14:23:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 599 : optimizer step time 0.98 +[Apr 20, 2026 - 14:23:01 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 599-------------------- +[Apr 20, 2026 - 14:23:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0093 iter count: 1.0 +[Apr 20, 2026 - 14:23:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0093 iter count: 1.0 +[Apr 20, 2026 - 14:23:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 14:23:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 14:26:17 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.339337 43.455994 43.207832 +peak_gpu_mem_gb 129.547819 131.549380 123.547254 +peak_gpu_mem_reserved_gb 133.519531 135.554688 127.488281 +util 65.500000 70.000000 60.000000 +[Apr 20, 2026 - 14:26:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 600 : avg iteration time 195.73 seconds +[Apr 20, 2026 - 14:26:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 600 : data loading time 0.00 +[Apr 20, 2026 - 14:26:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 600 : avg forward pass time 89.17 +[Apr 20, 2026 - 14:26:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 600 : backward pass time 8.80 +[Apr 20, 2026 - 14:26:17 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 600 : optimizer step time 0.34 +[Apr 20, 2026 - 14:26:17 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 600-------------------- +[Apr 20, 2026 - 14:26:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 8699.0510 iter count: 1.0 +[Apr 20, 2026 - 14:26:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5570 iter count: 1.0 +[Apr 20, 2026 - 14:26:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 8699.0510iter count: 1.0 +[Apr 20, 2026 - 14:26:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 14:26:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.7454iter count: 1.0 +[Apr 20, 2026 - 14:26:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.1744 iter count: 1.0 +[Apr 20, 2026 - 14:26:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 5.9664 iter count: 1.0 +[Apr 20, 2026 - 14:26:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 16645.7852iter count: 1.0 +[Apr 20, 2026 - 14:26:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 10.4722iter count: 1.0 +[Apr 20, 2026 - 14:26:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 152164.3281iter count: 1.0 +[Apr 20, 2026 - 14:26:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5570iter count: 1.0 +[Apr 20, 2026 - 14:26:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.1056 iter count: 1.0 +[Apr 20, 2026 - 14:26:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 3681.6589iter count: 1.0 +[Apr 20, 2026 - 14:26:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.3891 iter count: 1.0 +[Apr 20, 2026 - 14:26:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 878.5465iter count: 1.0 +[Apr 20, 2026 - 14:26:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 2.9981 iter count: 1.0 +[Apr 20, 2026 - 14:26:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 401.9311iter count: 1.0 +[Apr 20, 2026 - 14:26:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 3.2032 iter count: 1.0 +[Apr 20, 2026 - 14:26:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 605.7651iter count: 1.0 +[Apr 20, 2026 - 14:26:17 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[Apr 20, 2026 - 14:26:37 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 19.89 seconds +[Apr 20, 2026 - 14:26:37 | INFO | fastgen.trainer:save_checkpoint:266 ] Saving checkpoint iteration 600 +[Apr 20, 2026 - 14:26:37 | INFO | fastgen.utils.checkpointer:save:301 ] Saving FSDP model to prefix /tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints/0000600 +[Apr 20, 2026 - 14:26:53 | SUCCESS | fastgen.utils.checkpointer:save:329 ] Model saved at iteration 600 +[Apr 20, 2026 - 14:26:53 | INFO | fastgen.trainer:validate:358 ] Validation iteration 600 +[Apr 20, 2026 - 14:26:54 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 14:26:57 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 3.4s +[Apr 20, 2026 - 14:27:01 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 14:27:04 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 2.9s +[Apr 20, 2026 - 14:27:07 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 14:27:10 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 3.0s +[Apr 20, 2026 - 14:27:13 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 14:27:16 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 3.0s +[Apr 20, 2026 - 14:27:20 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 14:27:23 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 3.2s +[Apr 20, 2026 - 14:27:26 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 14:27:29 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 3.2s +[Apr 20, 2026 - 14:27:33 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 14:27:36 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 2.9s +[Apr 20, 2026 - 14:27:39 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 14:27:42 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 2.9s +[Apr 20, 2026 - 14:27:45 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 14:27:48 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 2.9s +[Apr 20, 2026 - 14:27:51 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:185 ] [val] Starting CausVid AR inference (B=1, T=21, steps=4) +[Apr 20, 2026 - 14:27:54 | INFO | fastgen.methods.omniavatar_self_forcing:validation_step:203 ] [val] AR inference done in 2.9s +[Apr 20, 2026 - 14:27:58 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging val0 stats at iteration 600-------------------- +[Apr 20, 2026 - 14:27:58 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0000 iter count: 10.0 +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +/home/work/.local/miniconda3/envs/hb_fastgen/lib/python3.12/site-packages/torchvision/io/_video_deprecation_warning.py:9: UserWarning: The video decoding and encoding capabilities of torchvision are deprecated from version 0.22 and will be removed in version 0.24. We recommend that you migrate to TorchCodec, where we'll consolidate the future decoding/encoding capabilities of PyTorch: https://github.com/pytorch/torchcodec + warnings.warn( +wandb: WARNING `fps` argument does not affect the frame rate of the video when providing a file path or raw bytes. +[Apr 20, 2026 - 14:28:14 | INFO | fastgen.callbacks.wandb:on_validation_end:683 ] Logged 10 val videos at iteration 600 +[Apr 20, 2026 - 14:29:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.339988 43.460228 43.207218 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.001465 70.388672 69.703125 +util 97.000000 100.000000 93.000000 +[Apr 20, 2026 - 14:29:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 601 : avg iteration time 166.57 seconds +[Apr 20, 2026 - 14:29:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 601 : data loading time 0.00 +[Apr 20, 2026 - 14:29:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 601 : avg forward pass time 12.21 +[Apr 20, 2026 - 14:29:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 601 : backward pass time 11.56 +[Apr 20, 2026 - 14:29:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 601 : optimizer step time 1.09 +[Apr 20, 2026 - 14:29:04 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 601-------------------- +[Apr 20, 2026 - 14:29:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0100 iter count: 1.0 +[Apr 20, 2026 - 14:29:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0100 iter count: 1.0 +[Apr 20, 2026 - 14:29:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 14:29:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 14:29:59 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.340069 43.460472 43.207203 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.020996 70.330078 69.761719 +util 97.250000 100.000000 93.000000 +[Apr 20, 2026 - 14:29:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 602 : avg iteration time 55.60 seconds +[Apr 20, 2026 - 14:29:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 602 : data loading time 0.00 +[Apr 20, 2026 - 14:29:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 602 : avg forward pass time 15.71 +[Apr 20, 2026 - 14:29:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 602 : backward pass time 11.62 +[Apr 20, 2026 - 14:29:59 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 602 : optimizer step time 0.97 +[Apr 20, 2026 - 14:29:59 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 602-------------------- +[Apr 20, 2026 - 14:29:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0105 iter count: 1.0 +[Apr 20, 2026 - 14:29:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0105 iter count: 1.0 +[Apr 20, 2026 - 14:29:59 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 14:29:59 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 14:30:53 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.340075 43.460468 43.207203 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.094238 70.369141 69.917969 +util 96.500000 100.000000 92.000000 +[Apr 20, 2026 - 14:30:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 603 : avg iteration time 53.39 seconds +[Apr 20, 2026 - 14:30:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 603 : data loading time 0.00 +[Apr 20, 2026 - 14:30:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 603 : avg forward pass time 14.01 +[Apr 20, 2026 - 14:30:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 603 : backward pass time 11.58 +[Apr 20, 2026 - 14:30:53 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 603 : optimizer step time 0.99 +[Apr 20, 2026 - 14:30:53 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 603-------------------- +[Apr 20, 2026 - 14:30:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0098 iter count: 1.0 +[Apr 20, 2026 - 14:30:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0098 iter count: 1.0 +[Apr 20, 2026 - 14:30:53 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 14:30:53 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 14:31:47 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.339355 43.457603 43.207211 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.045410 70.349609 69.742188 +util 94.250000 99.000000 91.000000 +[Apr 20, 2026 - 14:31:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 604 : avg iteration time 54.07 seconds +[Apr 20, 2026 - 14:31:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 604 : data loading time 0.00 +[Apr 20, 2026 - 14:31:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 604 : avg forward pass time 14.22 +[Apr 20, 2026 - 14:31:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 604 : backward pass time 12.83 +[Apr 20, 2026 - 14:31:47 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 604 : optimizer step time 1.22 +[Apr 20, 2026 - 14:31:47 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 604-------------------- +[Apr 20, 2026 - 14:31:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0117 iter count: 1.0 +[Apr 20, 2026 - 14:31:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[Apr 20, 2026 - 14:31:47 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 14:31:47 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 14:34:52 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.339399 43.456345 43.207661 +peak_gpu_mem_gb 129.547819 131.549380 123.547254 +peak_gpu_mem_reserved_gb 133.541016 135.576172 127.570312 +util 66.750000 70.000000 64.000000 +[Apr 20, 2026 - 14:34:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 605 : avg iteration time 185.78 seconds +[Apr 20, 2026 - 14:34:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 605 : data loading time 0.00 +[Apr 20, 2026 - 14:34:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 605 : avg forward pass time 84.34 +[Apr 20, 2026 - 14:34:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 605 : backward pass time 8.39 +[Apr 20, 2026 - 14:34:52 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 605 : optimizer step time 0.33 +[Apr 20, 2026 - 14:34:52 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 605-------------------- +[Apr 20, 2026 - 14:34:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 101388.2280 iter count: 1.0 +[Apr 20, 2026 - 14:34:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6900 iter count: 1.0 +[Apr 20, 2026 - 14:34:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 101388.2280iter count: 1.0 +[Apr 20, 2026 - 14:34:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 14:34:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.1771iter count: 1.0 +[Apr 20, 2026 - 14:34:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 1.7473 iter count: 1.0 +[Apr 20, 2026 - 14:34:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.4494 iter count: 1.0 +[Apr 20, 2026 - 14:34:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 122574.4219iter count: 1.0 +[Apr 20, 2026 - 14:34:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 32.9357iter count: 1.0 +[Apr 20, 2026 - 14:34:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 2954516.7500iter count: 1.0 +[Apr 20, 2026 - 14:34:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6900iter count: 1.0 +[Apr 20, 2026 - 14:34:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 5.5844 iter count: 1.0 +[Apr 20, 2026 - 14:34:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 70882.3359iter count: 1.0 +[Apr 20, 2026 - 14:34:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 7.4494 iter count: 1.0 +[Apr 20, 2026 - 14:34:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 2954516.7500iter count: 1.0 +[Apr 20, 2026 - 14:34:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 1.7473 iter count: 1.0 +[Apr 20, 2026 - 14:34:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 32.9357 iter count: 1.0 +[Apr 20, 2026 - 14:34:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 2.4558 iter count: 1.0 +[Apr 20, 2026 - 14:34:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 135.8514iter count: 1.0 +[Apr 20, 2026 - 14:34:52 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[Apr 20, 2026 - 14:34:52 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 14:35:44 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.339446 43.456596 43.207680 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.889160 70.035156 69.646484 +util 97.750000 100.000000 92.000000 +[Apr 20, 2026 - 14:35:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 606 : avg iteration time 51.89 seconds +[Apr 20, 2026 - 14:35:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 606 : data loading time 0.00 +[Apr 20, 2026 - 14:35:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 606 : avg forward pass time 13.29 +[Apr 20, 2026 - 14:35:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 606 : backward pass time 11.55 +[Apr 20, 2026 - 14:35:44 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 606 : optimizer step time 1.31 +[Apr 20, 2026 - 14:35:44 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 606-------------------- +[Apr 20, 2026 - 14:35:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0125 iter count: 1.0 +[Apr 20, 2026 - 14:35:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0125 iter count: 1.0 +[Apr 20, 2026 - 14:35:44 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 14:35:44 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 14:36:43 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.339447 43.456608 43.207661 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.845215 70.171875 69.333984 +util 98.000000 100.000000 93.000000 +[Apr 20, 2026 - 14:36:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 607 : avg iteration time 58.43 seconds +[Apr 20, 2026 - 14:36:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 607 : data loading time 0.00 +[Apr 20, 2026 - 14:36:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 607 : avg forward pass time 17.16 +[Apr 20, 2026 - 14:36:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 607 : backward pass time 11.56 +[Apr 20, 2026 - 14:36:43 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 607 : optimizer step time 1.00 +[Apr 20, 2026 - 14:36:43 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 607-------------------- +[Apr 20, 2026 - 14:36:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0112 iter count: 1.0 +[Apr 20, 2026 - 14:36:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[Apr 20, 2026 - 14:36:43 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 14:36:43 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 14:37:39 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.339460 43.456627 43.207672 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.830566 70.171875 69.451172 +util 99.750000 100.000000 99.000000 +[Apr 20, 2026 - 14:37:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 608 : avg iteration time 56.33 seconds +[Apr 20, 2026 - 14:37:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 608 : data loading time 0.00 +[Apr 20, 2026 - 14:37:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 608 : avg forward pass time 15.27 +[Apr 20, 2026 - 14:37:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 608 : backward pass time 11.57 +[Apr 20, 2026 - 14:37:39 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 608 : optimizer step time 1.32 +[Apr 20, 2026 - 14:37:39 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 608-------------------- +[Apr 20, 2026 - 14:37:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0125 iter count: 1.0 +[Apr 20, 2026 - 14:37:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0125 iter count: 1.0 +[Apr 20, 2026 - 14:37:39 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 14:37:39 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 14:38:40 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.339458 43.456615 43.207672 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.864746 70.035156 69.685547 +util 97.750000 100.000000 93.000000 +[Apr 20, 2026 - 14:38:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 609 : avg iteration time 60.87 seconds +[Apr 20, 2026 - 14:38:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 609 : data loading time 0.00 +[Apr 20, 2026 - 14:38:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 609 : avg forward pass time 18.38 +[Apr 20, 2026 - 14:38:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 609 : backward pass time 11.54 +[Apr 20, 2026 - 14:38:40 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 609 : optimizer step time 0.97 +[Apr 20, 2026 - 14:38:40 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 609-------------------- +[Apr 20, 2026 - 14:38:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0126 iter count: 1.0 +[Apr 20, 2026 - 14:38:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0126 iter count: 1.0 +[Apr 20, 2026 - 14:38:40 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 14:38:40 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 14:41:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338927 43.454514 43.207672 +peak_gpu_mem_gb 129.547819 131.549380 123.547254 +peak_gpu_mem_reserved_gb 133.524902 135.593750 127.566406 +util 72.500000 75.000000 70.000000 +[Apr 20, 2026 - 14:41:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 610 : avg iteration time 177.75 seconds +[Apr 20, 2026 - 14:41:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 610 : data loading time 0.00 +[Apr 20, 2026 - 14:41:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 610 : avg forward pass time 80.06 +[Apr 20, 2026 - 14:41:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 610 : backward pass time 8.42 +[Apr 20, 2026 - 14:41:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 610 : optimizer step time 0.34 +[Apr 20, 2026 - 14:41:38 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 610-------------------- +[Apr 20, 2026 - 14:41:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 43889.9335 iter count: 1.0 +[Apr 20, 2026 - 14:41:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5617 iter count: 1.0 +[Apr 20, 2026 - 14:41:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 43889.9335iter count: 1.0 +[Apr 20, 2026 - 14:41:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 14:41:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.8394iter count: 1.0 +[Apr 20, 2026 - 14:41:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.8287 iter count: 1.0 +[Apr 20, 2026 - 14:41:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 6.8352 iter count: 1.0 +[Apr 20, 2026 - 14:41:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 55429.8828iter count: 1.0 +[Apr 20, 2026 - 14:41:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 5.2455 iter count: 1.0 +[Apr 20, 2026 - 14:41:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 864895.0000iter count: 1.0 +[Apr 20, 2026 - 14:41:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5617iter count: 1.0 +[Apr 20, 2026 - 14:41:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 3.7439 iter count: 1.0 +[Apr 20, 2026 - 14:41:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 1786.1432iter count: 1.0 +[Apr 20, 2026 - 14:41:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 2.6653 iter count: 1.0 +[Apr 20, 2026 - 14:41:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 206.5609iter count: 1.0 +[Apr 20, 2026 - 14:41:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.3359 iter count: 1.0 +[Apr 20, 2026 - 14:41:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 5836.5771iter count: 1.0 +[Apr 20, 2026 - 14:41:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 5.8043 iter count: 1.0 +[Apr 20, 2026 - 14:41:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 110042.0469iter count: 1.0 +[Apr 20, 2026 - 14:41:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[Apr 20, 2026 - 14:41:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 14:42:38 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338993 43.454735 43.207695 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 70.538574 72.339844 69.646484 +util 96.000000 99.000000 92.000000 +[Apr 20, 2026 - 14:42:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 611 : avg iteration time 60.31 seconds +[Apr 20, 2026 - 14:42:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 611 : data loading time 0.00 +[Apr 20, 2026 - 14:42:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 611 : avg forward pass time 17.52 +[Apr 20, 2026 - 14:42:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 611 : backward pass time 11.56 +[Apr 20, 2026 - 14:42:38 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 611 : optimizer step time 0.97 +[Apr 20, 2026 - 14:42:38 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 611-------------------- +[Apr 20, 2026 - 14:42:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0116 iter count: 1.0 +[Apr 20, 2026 - 14:42:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0116 iter count: 1.0 +[Apr 20, 2026 - 14:42:38 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 14:42:38 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 14:43:32 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.339237 43.455723 43.207680 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.894043 70.074219 69.666016 +util 97.250000 100.000000 94.000000 +[Apr 20, 2026 - 14:43:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 612 : avg iteration time 54.25 seconds +[Apr 20, 2026 - 14:43:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 612 : data loading time 0.00 +[Apr 20, 2026 - 14:43:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 612 : avg forward pass time 14.25 +[Apr 20, 2026 - 14:43:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 612 : backward pass time 12.84 +[Apr 20, 2026 - 14:43:32 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 612 : optimizer step time 1.32 +[Apr 20, 2026 - 14:43:32 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 612-------------------- +[Apr 20, 2026 - 14:43:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0129 iter count: 1.0 +[Apr 20, 2026 - 14:43:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0129 iter count: 1.0 +[Apr 20, 2026 - 14:43:32 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 14:43:32 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 14:44:25 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.339293 43.455967 43.207680 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.889160 70.074219 69.626953 +util 95.500000 100.000000 92.000000 +[Apr 20, 2026 - 14:44:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 613 : avg iteration time 52.97 seconds +[Apr 20, 2026 - 14:44:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 613 : data loading time 0.00 +[Apr 20, 2026 - 14:44:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 613 : avg forward pass time 14.39 +[Apr 20, 2026 - 14:44:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 613 : backward pass time 11.57 +[Apr 20, 2026 - 14:44:25 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 613 : optimizer step time 1.02 +[Apr 20, 2026 - 14:44:25 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 613-------------------- +[Apr 20, 2026 - 14:44:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0108 iter count: 1.0 +[Apr 20, 2026 - 14:44:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0108 iter count: 1.0 +[Apr 20, 2026 - 14:44:25 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 14:44:25 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 14:45:16 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.339310 43.455990 43.207680 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.874512 70.015625 69.646484 +util 96.250000 99.000000 92.000000 +[Apr 20, 2026 - 14:45:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 614 : avg iteration time 50.54 seconds +[Apr 20, 2026 - 14:45:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 614 : data loading time 0.00 +[Apr 20, 2026 - 14:45:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 614 : avg forward pass time 12.77 +[Apr 20, 2026 - 14:45:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 614 : backward pass time 11.58 +[Apr 20, 2026 - 14:45:16 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 614 : optimizer step time 0.97 +[Apr 20, 2026 - 14:45:16 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 614-------------------- +[Apr 20, 2026 - 14:45:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0110 iter count: 1.0 +[Apr 20, 2026 - 14:45:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0110 iter count: 1.0 +[Apr 20, 2026 - 14:45:16 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 14:45:16 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 14:48:24 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.339060 43.455013 43.207680 +peak_gpu_mem_gb 129.547819 131.549380 123.547254 +peak_gpu_mem_reserved_gb 133.471680 135.576172 127.451172 +util 67.500000 70.000000 65.000000 +[Apr 20, 2026 - 14:48:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 615 : avg iteration time 187.88 seconds +[Apr 20, 2026 - 14:48:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 615 : data loading time 0.00 +[Apr 20, 2026 - 14:48:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 615 : avg forward pass time 85.40 +[Apr 20, 2026 - 14:48:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 615 : backward pass time 8.39 +[Apr 20, 2026 - 14:48:24 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 615 : optimizer step time 0.34 +[Apr 20, 2026 - 14:48:24 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 615-------------------- +[Apr 20, 2026 - 14:48:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 119318.2761 iter count: 1.0 +[Apr 20, 2026 - 14:48:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6041 iter count: 1.0 +[Apr 20, 2026 - 14:48:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 119318.2761iter count: 1.0 +[Apr 20, 2026 - 14:48:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 14:48:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.1143iter count: 1.0 +[Apr 20, 2026 - 14:48:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.5710 iter count: 1.0 +[Apr 20, 2026 - 14:48:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.5482 iter count: 1.0 +[Apr 20, 2026 - 14:48:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 152224.4219iter count: 1.0 +[Apr 20, 2026 - 14:48:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 3.1328 iter count: 1.0 +[Apr 20, 2026 - 14:48:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 3599910.0000iter count: 1.0 +[Apr 20, 2026 - 14:48:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6041iter count: 1.0 +[Apr 20, 2026 - 14:48:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 5.8936 iter count: 1.0 +[Apr 20, 2026 - 14:48:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 131550.5625iter count: 1.0 +[Apr 20, 2026 - 14:48:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.7966 iter count: 1.0 +[Apr 20, 2026 - 14:48:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 1984.4626iter count: 1.0 +[Apr 20, 2026 - 14:48:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 6.6160 iter count: 1.0 +[Apr 20, 2026 - 14:48:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 557940.6875iter count: 1.0 +[Apr 20, 2026 - 14:48:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 2.8383 iter count: 1.0 +[Apr 20, 2026 - 14:48:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 291.9705iter count: 1.0 +[Apr 20, 2026 - 14:48:24 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0121 iter count: 1.0 +[Apr 20, 2026 - 14:48:24 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 14:49:18 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.339128 43.455276 43.207687 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.928223 70.171875 69.646484 +util 97.500000 99.000000 95.000000 +[Apr 20, 2026 - 14:49:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 616 : avg iteration time 54.41 seconds +[Apr 20, 2026 - 14:49:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 616 : data loading time 0.00 +[Apr 20, 2026 - 14:49:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 616 : avg forward pass time 14.61 +[Apr 20, 2026 - 14:49:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 616 : backward pass time 11.52 +[Apr 20, 2026 - 14:49:18 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 616 : optimizer step time 0.99 +[Apr 20, 2026 - 14:49:18 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 616-------------------- +[Apr 20, 2026 - 14:49:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0128 iter count: 1.0 +[Apr 20, 2026 - 14:49:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0128 iter count: 1.0 +[Apr 20, 2026 - 14:49:18 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 14:49:18 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 14:50:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.339134 43.455276 43.207710 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.845215 70.132812 69.449219 +util 96.750000 99.000000 93.000000 +[Apr 20, 2026 - 14:50:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 617 : avg iteration time 54.12 seconds +[Apr 20, 2026 - 14:50:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 617 : data loading time 0.00 +[Apr 20, 2026 - 14:50:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 617 : avg forward pass time 14.32 +[Apr 20, 2026 - 14:50:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 617 : backward pass time 12.85 +[Apr 20, 2026 - 14:50:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 617 : optimizer step time 1.09 +[Apr 20, 2026 - 14:50:12 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 617-------------------- +[Apr 20, 2026 - 14:50:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0143 iter count: 1.0 +[Apr 20, 2026 - 14:50:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0143 iter count: 1.0 +[Apr 20, 2026 - 14:50:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 14:50:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 14:51:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.339134 43.455280 43.207706 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.957520 70.113281 69.783203 +util 95.750000 99.000000 92.000000 +[Apr 20, 2026 - 14:51:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 618 : avg iteration time 52.65 seconds +[Apr 20, 2026 - 14:51:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 618 : data loading time 0.00 +[Apr 20, 2026 - 14:51:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 618 : avg forward pass time 14.26 +[Apr 20, 2026 - 14:51:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 618 : backward pass time 11.57 +[Apr 20, 2026 - 14:51:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 618 : optimizer step time 0.98 +[Apr 20, 2026 - 14:51:05 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 618-------------------- +[Apr 20, 2026 - 14:51:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0127 iter count: 1.0 +[Apr 20, 2026 - 14:51:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0127 iter count: 1.0 +[Apr 20, 2026 - 14:51:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 14:51:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 14:52:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.339149 43.455280 43.207706 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.957520 70.152344 69.783203 +util 96.750000 100.000000 93.000000 +[Apr 20, 2026 - 14:52:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 619 : avg iteration time 59.15 seconds +[Apr 20, 2026 - 14:52:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 619 : data loading time 0.00 +[Apr 20, 2026 - 14:52:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 619 : avg forward pass time 16.66 +[Apr 20, 2026 - 14:52:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 619 : backward pass time 11.96 +[Apr 20, 2026 - 14:52:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 619 : optimizer step time 1.56 +[Apr 20, 2026 - 14:52:04 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 619-------------------- +[Apr 20, 2026 - 14:52:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0148 iter count: 1.0 +[Apr 20, 2026 - 14:52:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0148 iter count: 1.0 +[Apr 20, 2026 - 14:52:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 14:52:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 14:55:01 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.339132 43.455265 43.207710 +peak_gpu_mem_gb 129.547819 131.549380 123.547254 +peak_gpu_mem_reserved_gb 133.476562 135.517578 127.509766 +util 69.750000 73.000000 67.000000 +[Apr 20, 2026 - 14:55:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 620 : avg iteration time 177.30 seconds +[Apr 20, 2026 - 14:55:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 620 : data loading time 0.00 +[Apr 20, 2026 - 14:55:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 620 : avg forward pass time 80.05 +[Apr 20, 2026 - 14:55:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 620 : backward pass time 8.43 +[Apr 20, 2026 - 14:55:01 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 620 : optimizer step time 0.35 +[Apr 20, 2026 - 14:55:01 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 620-------------------- +[Apr 20, 2026 - 14:55:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 4527.7298 iter count: 1.0 +[Apr 20, 2026 - 14:55:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.5729 iter count: 1.0 +[Apr 20, 2026 - 14:55:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 4527.7298iter count: 1.0 +[Apr 20, 2026 - 14:55:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 14:55:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 3.5599iter count: 1.0 +[Apr 20, 2026 - 14:55:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.8022 iter count: 1.0 +[Apr 20, 2026 - 14:55:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 5.6072 iter count: 1.0 +[Apr 20, 2026 - 14:55:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 8818.5068iter count: 1.0 +[Apr 20, 2026 - 14:55:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 4.9752 iter count: 1.0 +[Apr 20, 2026 - 14:55:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 74195.6875iter count: 1.0 +[Apr 20, 2026 - 14:55:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.5729iter count: 1.0 +[Apr 20, 2026 - 14:55:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.0355 iter count: 1.0 +[Apr 20, 2026 - 14:55:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 3200.2612iter count: 1.0 +[Apr 20, 2026 - 14:55:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 3.1308 iter count: 1.0 +[Apr 20, 2026 - 14:55:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 524.0780iter count: 1.0 +[Apr 20, 2026 - 14:55:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 4.6105 iter count: 1.0 +[Apr 20, 2026 - 14:55:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 10107.9248iter count: 1.0 +[Apr 20, 2026 - 14:55:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.2149 iter count: 1.0 +[Apr 20, 2026 - 14:55:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 4581.5229iter count: 1.0 +[Apr 20, 2026 - 14:55:01 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0120 iter count: 1.0 +[Apr 20, 2026 - 14:55:01 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 14:56:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.339196 43.455521 43.207710 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.894043 69.996094 69.822266 +util 97.750000 100.000000 95.000000 +[Apr 20, 2026 - 14:56:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 621 : avg iteration time 58.39 seconds +[Apr 20, 2026 - 14:56:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 621 : data loading time 0.00 +[Apr 20, 2026 - 14:56:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 621 : avg forward pass time 16.65 +[Apr 20, 2026 - 14:56:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 621 : backward pass time 11.57 +[Apr 20, 2026 - 14:56:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 621 : optimizer step time 0.97 +[Apr 20, 2026 - 14:56:00 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 621-------------------- +[Apr 20, 2026 - 14:56:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0147 iter count: 1.0 +[Apr 20, 2026 - 14:56:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0147 iter count: 1.0 +[Apr 20, 2026 - 14:56:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 14:56:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 14:57:04 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.339145 43.455273 43.207710 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.908691 70.035156 69.822266 +util 96.750000 99.000000 93.000000 +[Apr 20, 2026 - 14:57:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 622 : avg iteration time 64.53 seconds +[Apr 20, 2026 - 14:57:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 622 : data loading time 0.00 +[Apr 20, 2026 - 14:57:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 622 : avg forward pass time 19.84 +[Apr 20, 2026 - 14:57:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 622 : backward pass time 12.31 +[Apr 20, 2026 - 14:57:04 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 622 : optimizer step time 0.99 +[Apr 20, 2026 - 14:57:04 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 622-------------------- +[Apr 20, 2026 - 14:57:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0141 iter count: 1.0 +[Apr 20, 2026 - 14:57:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0141 iter count: 1.0 +[Apr 20, 2026 - 14:57:04 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 14:57:04 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 14:58:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.339149 43.455261 43.207737 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.889160 69.976562 69.841797 +util 94.750000 99.000000 91.000000 +[Apr 20, 2026 - 14:58:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 623 : avg iteration time 56.04 seconds +[Apr 20, 2026 - 14:58:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 623 : data loading time 0.00 +[Apr 20, 2026 - 14:58:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 623 : avg forward pass time 15.48 +[Apr 20, 2026 - 14:58:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 623 : backward pass time 11.60 +[Apr 20, 2026 - 14:58:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 623 : optimizer step time 0.98 +[Apr 20, 2026 - 14:58:00 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 623-------------------- +[Apr 20, 2026 - 14:58:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0128 iter count: 1.0 +[Apr 20, 2026 - 14:58:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0128 iter count: 1.0 +[Apr 20, 2026 - 14:58:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 14:58:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 14:59:00 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.339159 43.455284 43.207714 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.864746 69.898438 69.839844 +util 97.500000 100.000000 91.000000 +[Apr 20, 2026 - 14:59:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 624 : avg iteration time 59.55 seconds +[Apr 20, 2026 - 14:59:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 624 : data loading time 0.00 +[Apr 20, 2026 - 14:59:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 624 : avg forward pass time 16.93 +[Apr 20, 2026 - 14:59:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 624 : backward pass time 12.80 +[Apr 20, 2026 - 14:59:00 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 624 : optimizer step time 1.29 +[Apr 20, 2026 - 14:59:00 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 624-------------------- +[Apr 20, 2026 - 14:59:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0117 iter count: 1.0 +[Apr 20, 2026 - 14:59:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0117 iter count: 1.0 +[Apr 20, 2026 - 14:59:00 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 14:59:00 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 15:02:14 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338848 43.454056 43.207729 +peak_gpu_mem_gb 129.547819 131.549380 123.547254 +peak_gpu_mem_reserved_gb 133.491699 135.537109 127.509766 +util 78.500000 96.000000 64.000000 +[Apr 20, 2026 - 15:02:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 625 : avg iteration time 194.58 seconds +[Apr 20, 2026 - 15:02:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 625 : data loading time 0.00 +[Apr 20, 2026 - 15:02:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 625 : avg forward pass time 88.18 +[Apr 20, 2026 - 15:02:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 625 : backward pass time 9.53 +[Apr 20, 2026 - 15:02:14 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 625 : optimizer step time 0.34 +[Apr 20, 2026 - 15:02:14 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 625-------------------- +[Apr 20, 2026 - 15:02:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 73196.8325 iter count: 1.0 +[Apr 20, 2026 - 15:02:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss: 0.6205 iter count: 1.0 +[Apr 20, 2026 - 15:02:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_weighted: 73196.8325iter count: 1.0 +[Apr 20, 2026 - 15:02:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_gen: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 15:02:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_mean: 4.1869iter count: 1.0 +[Apr 20, 2026 - 15:02:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_min: 0.7072 iter count: 1.0 +[Apr 20, 2026 - 15:02:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_max: 7.0989 iter count: 1.0 +[Apr 20, 2026 - 15:02:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_mean: 106696.4375iter count: 1.0 +[Apr 20, 2026 - 15:02:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_min: 4.1139 iter count: 1.0 +[Apr 20, 2026 - 15:02:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_max: 1465761.6250iter count: 1.0 +[Apr 20, 2026 - 15:02:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_vsd_loss_unweighted: 0.6205iter count: 1.0 +[Apr 20, 2026 - 15:02:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r0: 4.0426 iter count: 1.0 +[Apr 20, 2026 - 15:02:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r0: 3245.7944iter count: 1.0 +[Apr 20, 2026 - 15:02:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r1: 4.6336 iter count: 1.0 +[Apr 20, 2026 - 15:02:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r1: 10584.3301iter count: 1.0 +[Apr 20, 2026 - 15:02:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r2: 5.7842 iter count: 1.0 +[Apr 20, 2026 - 15:02:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r2: 105695.8906iter count: 1.0 +[Apr 20, 2026 - 15:02:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_sync_c_r3: 4.1337 iter count: 1.0 +[Apr 20, 2026 - 15:02:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_reward_weight_r3: 3895.0425iter count: 1.0 +[Apr 20, 2026 - 15:02:14 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0112 iter count: 1.0 +[Apr 20, 2026 - 15:02:14 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.01 seconds +[Apr 20, 2026 - 15:03:12 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.338972 43.454533 43.207714 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.869629 69.957031 69.646484 +util 97.250000 100.000000 94.000000 +[Apr 20, 2026 - 15:03:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 626 : avg iteration time 57.74 seconds +[Apr 20, 2026 - 15:03:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 626 : data loading time 0.00 +[Apr 20, 2026 - 15:03:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 626 : avg forward pass time 16.81 +[Apr 20, 2026 - 15:03:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 626 : backward pass time 11.54 +[Apr 20, 2026 - 15:03:12 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 626 : optimizer step time 1.03 +[Apr 20, 2026 - 15:03:12 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 626-------------------- +[Apr 20, 2026 - 15:03:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0132 iter count: 1.0 +[Apr 20, 2026 - 15:03:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0132 iter count: 1.0 +[Apr 20, 2026 - 15:03:12 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 15:03:12 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds +[Apr 20, 2026 - 15:04:05 | INFO | fastgen.callbacks.gpu_stats:log_prof_data:45 ] GPU stats: + Avg Max Min +cpu_mem_gb 43.339035 43.454765 43.207714 +peak_gpu_mem_gb 67.807382 67.808927 67.806867 +peak_gpu_mem_reserved_gb 69.879395 70.015625 69.705078 +util 96.750000 99.000000 90.000000 +[Apr 20, 2026 - 15:04:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:110 ] 627 : avg iteration time 53.25 seconds +[Apr 20, 2026 - 15:04:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:123 ] 627 : data loading time 0.00 +[Apr 20, 2026 - 15:04:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:124 ] 627 : avg forward pass time 13.94 +[Apr 20, 2026 - 15:04:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:125 ] 627 : backward pass time 11.57 +[Apr 20, 2026 - 15:04:05 | INFO | fastgen.callbacks.train_profiler:on_training_step_end:126 ] 627 : optimizer step time 0.97 +[Apr 20, 2026 - 15:04:05 | INFO | fastgen.callbacks.wandb:log_stats:570 ] logging train stats at iteration 627-------------------- +[Apr 20, 2026 - 15:04:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_total_loss: 0.0123 iter count: 1.0 +[Apr 20, 2026 - 15:04:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_fake_score_loss: 0.0123 iter count: 1.0 +[Apr 20, 2026 - 15:04:05 | INFO | fastgen.callbacks.wandb:get_stat:304 ] avg_gan_loss_disc: 0.0000 iter count: 1.0 +[Apr 20, 2026 - 15:04:05 | INFO | fastgen.callbacks.wandb:on_training_step_end:601 ] WandB logging complete after 0.00 seconds diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/requirements.txt b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..a7117b8203d3b4df2b52c765d49a822a97b57cd9 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/requirements.txt @@ -0,0 +1,247 @@ +nvitop==1.6.1 +ftfy==6.3.1 +braceexpand==0.1.7 +antlr4-python3-runtime==4.9.3 +webdataset==1.0.2 +sentry-sdk==2.53.0 +rdkit==2025.9.5 +python-dotenv==1.2.1 +proglog==0.1.12 +omegaconf==2.3.0 +narwhals==2.17.0 +loguru==0.7.3 +imageio-ffmpeg==0.6.0 +plotly==6.5.2 +moviepy==2.2.1 +hydra-core==1.3.2 +toml==0.10.2 +wandb==0.25.0 +types-certifi==2021.10.8.3 +types-toml==0.10.8.20260408 +synchronicity==0.12.2 +propcache==0.4.1 +multidict==6.7.1 +hyperframe==6.1.0 +hpack==4.1.0 +frozenlist==1.8.0 +cbor2==5.9.0 +aiohappyeyeballs==2.6.1 +yarl==1.23.0 +watchfiles==1.1.1 +h2==4.3.0 +aiosignal==1.4.0 +grpclib==0.4.9 +aiohttp==3.13.5 +modal==1.4.1 +packaging==25.0 +setuptools==80.10.2 +wheel==0.46.3 +pip==26.0.1 +webencodings==0.5.1 +pure_eval==0.2.3 +ptyprocess==0.7.0 +nvidia-ml-py==13.590.48 +nvidia-cusparselt-cu12==0.7.1 +mpmath==1.3.0 +fastjsonschema==2.21.2 +zipp==3.23.0 +xyzservices==2025.11.0 +widgetsnbextension==4.0.15 +websocket-client==1.9.0 +webcolors==25.10.0 +wcwidth==0.6.0 +urllib3==2.6.3 +uri-template==1.3.0 +tzdata==2025.3 +typing_extensions==4.15.0 +triton==3.6.0 +traitlets==5.14.3 +tqdm==4.67.3 +tornado==6.5.5 +tinycss2==1.4.0 +sympy==1.14.0 +soupsieve==2.8.3 +smmap==5.0.3 +six==1.16.0 +sentencepiece==0.2.1 +Send2Trash==2.1.0 +safetensors==0.7.0 +rpds-py==0.30.0 +rfc3986-validator==0.1.1 +regex==2026.2.28 +pyzmq==27.1.0 +PyYAML==6.0.3 +python-json-logger==4.0.0 +Pygments==2.19.2 +pycparser==3.0 +psutil==7.2.2 +protobuf==4.24.4 +prometheus_client==0.24.1 +platformdirs==4.9.4 +pillow==11.3.0 +pexpect==4.9.0 +parso==0.8.6 +pandocfilters==1.5.1 +nvidia-nvtx-cu12==12.8.90 +nvidia-nvshmem-cu12==3.4.5 +nvidia-nvjitlink-cu12==12.8.93 +nvidia-nccl-cu12==2.27.5 +nvidia-curand-cu12==10.3.9.90 +nvidia-cufile-cu12==1.13.1.3 +nvidia-cuda-runtime-cu12==12.8.90 +nvidia-cuda-nvrtc-cu12==12.8.93 +nvidia-cuda-cupti-cu12==12.8.90 +nvidia-cublas-cu12==12.8.4.1 +numpy==1.26.4 +networkx==3.6.1 +nest-asyncio==1.6.0 +mistune==3.2.0 +MarkupSafe==3.0.3 +lark==1.3.1 +jupyterlab_widgets==3.0.16 +jupyterlab_pygments==0.3.0 +jsonpointer==3.0.0 +json5==0.13.0 +jmespath==1.1.0 +idna==3.11 +hf-xet==1.4.2 +h11==0.16.0 +fsspec==2026.2.0 +fqdn==1.5.1 +filelock==3.25.2 +executing==2.2.1 +einops==0.8.2 +defusedxml==0.7.1 +decorator==5.2.1 +debugpy==1.8.20 +cuda-pathfinder==1.4.2 +comm==0.2.3 +charset-normalizer==3.4.5 +certifi==2026.2.25 +bleach==6.3.0 +babel==2.18.0 +av==17.0.0 +attrs==25.4.0 +async-lru==2.2.0 +asttokens==3.0.1 +annotated-types==0.7.0 +typing-inspection==0.4.2 +terminado==0.18.1 +stack-data==0.6.3 +scipy==1.17.1 +rfc3987-syntax==1.1.0 +rfc3339-validator==0.1.4 +requests==2.32.5 +referencing==0.37.0 +python-dateutil==2.9.0.post0 +pydantic_core==2.41.5 +prompt_toolkit==3.0.52 +opencv-python-headless==4.11.0.86 +nvidia-cusparse-cu12==12.5.8.93 +nvidia-cufft-cu12==11.3.3.83 +nvidia-cudnn-cu12==9.10.2.21 +matplotlib-inline==0.2.1 +jupyter_core==5.9.1 +Jinja2==3.1.6 +jedi==0.19.2 +ipython_pygments_lexers==1.1.1 +importlib_metadata==8.7.1 +ImageIO==2.37.3 +httpcore==1.0.9 +gitdb==4.0.12 +cuda-bindings==12.9.4 +contourpy==1.3.3 +cffi==2.0.0 +beautifulsoup4==4.14.3 +anyio==4.12.1 +soundfile==0.13.1 +pydantic==2.12.5 +nvidia-cusolver-cu12==11.7.3.90 +jupyter_server_terminals==0.5.4 +jupyter_client==8.8.0 +jsonschema-specifications==2025.9.1 +ipython==9.11.0 +httpx==0.28.1 +GitPython==3.1.46 +botocore==1.42.68 +bokeh==3.9.0 +arrow==1.4.0 +argon2-cffi-bindings==25.1.0 +torch==2.10.0 +s3transfer==0.16.0 +jsonschema==4.26.0 +isoduration==20.11.0 +ipywidgets==8.1.8 +ipykernel==7.2.0 +argon2-cffi==25.1.0 +torchvision==0.25.0 +nbformat==5.10.4 +jupyter-console==6.6.3 +boto3==1.42.68 +accelerate==1.13.0 +nbclient==0.10.4 +jupyter-events==0.12.0 +nbconvert==7.17.0 +jupyter_server==2.17.0 +notebook_shim==0.2.4 +jupyterlab_server==2.28.0 +jupyter-lsp==2.3.0 +jupyterlab==4.5.6 +notebook==7.5.5 +jupyter==1.1.1 +fastgen==0.1.0 +pandas==3.0.1 +shellingham==1.5.4 +mdurl==0.1.2 +annotated-doc==0.0.4 +markdown-it-py==4.0.0 +rich==14.3.3 +typer==0.24.1 +huggingface_hub==1.7.1 +timm==1.0.25 +tokenizers==0.22.2 +diffusers==0.37.0 +transformers==5.3.0 +peft==0.18.1 +easydict==1.13 +lmdb==2.2.0 +threadpoolctl==3.6.0 +soxr==1.0.0 +msgpack==1.1.2 +llvmlite==0.47.0 +lazy-loader==0.5 +joblib==1.5.3 +audioread==3.1.0 +scikit-learn==1.8.0 +pooch==1.9.0 +numba==0.65.0 +librosa==0.11.0 +simsimd==6.5.16 +flatbuffers==25.12.19 +tifffile==2026.3.3 +stringzilla==4.6.0 +pyparsing==3.3.2 +prettytable==3.17.0 +onnx==1.17.0 +kiwisolver==1.5.0 +fonttools==4.62.1 +Cython==3.2.4 +cycler==0.12.1 +scikit-image==0.26.0 +onnxruntime==1.24.4 +matplotlib==3.10.8 +albucore==0.0.24 +albumentations==2.0.8 +insightface==0.7.3 +kornia_rs==0.1.10 +kornia==0.8.2 +pluggy==1.6.0 +iniconfig==2.3.0 +pytest==8.1.1 +dlib==20.0.1 +lpips==0.1.4 +python_speech_features==0.6 +decord==0.6.0 +click==8.2.1 +scenedetect==0.6.7.1 +torchaudio==2.10.0 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/wandb-metadata.json b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..b2040304d500f880471f939cc7b12be794a8e268 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/files/wandb-metadata.json @@ -0,0 +1,70 @@ +{ + "os": "Linux-5.15.0-151-generic-x86_64-with-glibc2.39", + "python": "CPython 3.12.12", + "startedAt": "2026-04-19T15:28:42.138669Z", + "args": [ + "--config=fastgen/configs/experiments/OmniAvatar/config_sf_sink1_window7_redmd_beta2_taew_4step.py", + "-", + "trainer.resume=False", + "log_config.group=omniavatar_sf_audiofix", + "log_config.name=sf_sink1_window7_redmd_audiofix_beta2_taew_4step", + "log_config.project=OmniAvatar-FastGen", + "log_config.wandb_entity=paulhcho" + ], + "program": "/home/work/.local/hyunbin/FastGen-redmd/train.py", + "codePath": "train.py", + "codePathLocal": "train.py", + "git": { + "remote": "https://paulcho98:@github.com/paulcho98/FastGen.git", + "commit": "9f0082e9f8af532da3aa92c5a238e609ea13d4d1" + }, + "email": "paul.hyunbin@gmail.com", + "root": "/tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step", + "host": "main1", + "executable": "/home/work/.local/miniconda3/envs/hb_fastgen/bin/python3.12", + "cpu_count": 112, + "cpu_count_logical": 224, + "gpu": "NVIDIA H200", + "gpu_count": 4, + "disk": { + "/": { + "total": "1356758433792", + "used": "877292904448" + } + }, + "memory": { + "total": "2163961778176" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-4685d4b3-5cf9-2766-43d3-b9615a684b7c" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ec888a66-4b6f-b8de-b34b-249efb9ad262" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-9c1e1773-d710-06c9-7db7-1b54e9fc3790" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-2b1017dc-2958-a946-16d2-2c29da6d18b0" + } + ], + "cudaVersion": "12.9", + "writerId": "c605u6zk7n0tk92tcvrzhi7ec1vz5s4b" +} \ No newline at end of file diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/logs/debug-core.log b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..1f03ded92d78bd558f3f32d2e73b42663e03cc71 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/logs/debug-core.log @@ -0,0 +1,8 @@ +{"time":"2026-04-20T00:28:42.196268529+09:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpylafhed8/port-2230976.txt","pid":2230976,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-04-20T00:28:42.19674978+09:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":2230976} +{"time":"2026-04-20T00:28:42.19673715+09:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-2230976-2256241-2440757848/socket","Net":"unix"}} +{"time":"2026-04-20T00:28:42.382924029+09:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-04-20T00:28:42.39791579+09:00","level":"INFO","msg":"handleInformInit: received","streamId":"fxpzd9v7","id":"1(@)"} +{"time":"2026-04-20T00:28:42.787670055+09:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"fxpzd9v7","id":"1(@)"} +{"time":"2026-04-20T00:28:48.787388183+09:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"v18g2qz28xvq"} +{"time":"2026-04-20T15:04:24.030855253+09:00","level":"INFO","msg":"server: parent process exited, terminating service process"} diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/logs/debug-internal.log b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..687e621fbafdd3d33cb9721ca2e68bd40f535263 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/logs/debug-internal.log @@ -0,0 +1,6 @@ +{"time":"2026-04-20T00:28:42.398011276+09:00","level":"INFO","msg":"stream: starting","core version":"0.25.0"} +{"time":"2026-04-20T00:28:42.787548846+09:00","level":"INFO","msg":"stream: created new stream","id":"fxpzd9v7"} +{"time":"2026-04-20T00:28:42.787597435+09:00","level":"INFO","msg":"handler: started","stream_id":"fxpzd9v7"} +{"time":"2026-04-20T00:28:42.787663397+09:00","level":"INFO","msg":"stream: started","id":"fxpzd9v7"} +{"time":"2026-04-20T00:28:42.78768401+09:00","level":"INFO","msg":"sender: started","stream_id":"fxpzd9v7"} +{"time":"2026-04-20T00:28:42.787687913+09:00","level":"INFO","msg":"writer: started","stream_id":"fxpzd9v7"} diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/logs/debug.log b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..b840db1d95308475597107b86844acbf528b90d9 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/logs/debug.log @@ -0,0 +1,19 @@ +2026-04-20 00:28:42,139 INFO MainThread:2230976 [wandb_setup.py:_flush():81] Current SDK version is 0.25.0 +2026-04-20 00:28:42,139 INFO MainThread:2230976 [wandb_setup.py:_flush():81] Configure stats pid to 2230976 +2026-04-20 00:28:42,139 INFO MainThread:2230976 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-04-20 00:28:42,140 INFO MainThread:2230976 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/logs/debug.log +2026-04-20 00:28:42,140 INFO MainThread:2230976 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/logs/debug-internal.log +2026-04-20 00:28:42,140 INFO MainThread:2230976 [wandb_init.py:init():844] calling init triggers +2026-04-20 00:28:42,140 INFO MainThread:2230976 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'log_config': {'project': 'OmniAvatar-FastGen', 'group': 'omniavatar_sf_audiofix', 'name': 'sf_sink1_window7_redmd_audiofix_beta2_taew_4step', 'wandb_mode': 'online', 'wandb_entity': 'paulhcho', 'wandb_credential': './credentials/wandb_api.txt'}, 'trainer': {'cudnn': {'deterministic': 'False', 'benchmark': 'True'}, 'checkpointer': {'save_dir': '/tmp/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/checkpoints', 'use_s3': 'False', 's3_container': 's3://checkpoints/fastgen', 's3_credential': './credentials/s3.json', 'pretrained_ckpt_path': '/home/work/.local/hyunbin/FastGen-redmd/FASTGEN_OUTPUT/OmniAvatar-FastGen/omniavatar_df_audiofix/df_audiofix_shift_5_4gpu_bs16_lr1e5_10000iter/checkpoints/0010000.pth', 'pretrained_ckpt_key_map': {'net': 'net'}}, 'callbacks': {'grad_clip': {'grad_norm': '10.0', 'model_key': 'net', '_target_': ""}, 'gpu_stats': {'every_n': '100', '_target_': ""}, 'train_profiler': {'every_n': '100', '_target_': ""}, 'param_count': {'_target_': ""}, 'ema': {'type': 'constant', 'beta': '0.9999', 'gamma': '16.97', 'ema_halflife_kimg': '500', 'ema_rampup_ratio': '0.05', 'start_iter': '0', '_target_': ""}, 'wandb': {'sample_logging_iter': '100', '_target_': "", 'fps': '25'}}, 'save_ckpt_iter': '100', 'validation_iter': '100', 'skip_initial_validation': 'True', 'logging_iter': '1', 'max_iter': '5000', 'visualize_teacher': 'False', 'seed': '0', 'val_seed': None, 'resume': 'False', 'ddp': 'False', 'fsdp': 'True', 'tf32_enabled': 'True', 'grad_accum_rounds': '2', 'batch_size_global': None, 'offload_module_in_decoding': 'False', 'fsdp_cpu_offload': 'False', 'fsdp_min_num_params': '10000000', 'fsdp_sharding_group_size': None, 'global_vars': None, 'global_vars_val': [None], 'augment_pipe': None}, 'dataloader_train': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_path.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '8', 'num_workers': '2', 'neg_text_emb_path': None, 'use_ref_sequence': 'True', '_target_': "", 'load_raw_audio': 'True', 'raw_audio_sample_rate': '16000', 'raw_audio_num_frames': '81', 'raw_audio_fps': '25.0'}, 'dataloader_val': {'data_list_path': '/home/work/stableavatar_data/v2v_training_data/video_square_val10.txt', 'latentsync_mask_path': '/home/work/.local/Self-Forcing_LipSync_StableAvatar/diffsynth/utils/mask.png', 'batch_size': '1', 'num_workers': '2', 'neg_text_emb_path': None, 'use_ref_sequence': 'True', 'load_ode_path': 'False', '_target_': ''}, 'eval': {'num_samples': '50000', 'save_images': 'False', 'min_ckpt': '0', 'max_ckpt': '100000000', 'samples_dir': 'samples'}, 'model': {'net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'chunk_size': '3', 'total_num_frames': '21', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_phase2/step-19500.pt', 'net_pred_type': 'flow', 'schedule_type': 'rf', 'local_attn_size': '7', 'sink_size': '1', 'use_dynamic_rope': 'True', '_target_': ""}, 'teacher': {'model_size': '14B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00001-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00002-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00003-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00004-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00005-of-00006.safetensors,/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-14B/diffusion_pytorch_model-00006-of-00006.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_phase2/step-10500.pt', 'merge_lora': 'True', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'fake_score_net': {'model_size': '1.3B', 'in_dim': '65', 'mode': 'v2v', 'use_audio': 'True', 'audio_hidden_size': '32', 'base_model_paths': '/home/work/.local/OmniAvatar/pretrained_models/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors', 'omniavatar_ckpt_path': '/home/work/output_omniavatar_v2v_1.3B_phase2/step-19500.pt', 'merge_lora': 'False', 'net_pred_type': 'flow', 'schedule_type': 'rf', '_target_': ""}, 'guidance_scale': '4.5', 'timestep_cfg': {'enabled': 'True', 't_lo': '0.556', 't_hi': '0.882'}, 'skip_layers': None, 'net_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'net_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'sample_t_cfg': {'time_dist_type': 'shifted', 'train_p_mean': '-1.1', 'train_p_std': '2.0', 'shift': '5.0', 'min_t': '0.001', 'max_t': '0.999', 't_list': ['0.999', '0.937', '0.833', '0.624', '0.0'], 'log_t_df': '0.01'}, 'input_shape': ['16', '21', '64', '64'], 'device': 'cuda', 'grad_scaler_enabled': 'False', 'grad_scaler_init_scale': '65536.0', 'grad_scaler_growth_interval': '2000', 'pretrained_model_path': '', 'pretrained_student_net_path': '', 'load_student_weights': 'False', 'enable_preprocessors': 'True', 'use_ema': 'False', 'student_sample_steps': '4', 'student_sample_type': 'sde', 'fsdp_meta_init': 'False', 'add_teacher_to_fsdp_dict': 'True', 'ddp_find_unused_parameters': 'True', 'precision': 'bfloat16', 'precision_amp': None, 'precision_amp_infer': None, 'precision_amp_enc': None, 'precision_fsdp': 'float32', 'fake_score_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '2e-06', 'weight_decay': '0.01', 'betas': ['0.0', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'fake_score_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'discriminator': {'feature_indices': '{0, 1, 2}', 'all_res': ['32', '16', '8'], 'in_channels': '256', '_target_': ""}, 'discriminator_optimizer': {'model': None, 'optim_type': 'adamw', 'lr': '0.0001', 'weight_decay': '0.01', 'betas': ['0.9', '0.999'], 'eps': '1e-08', 'fused': 'False', '_target_': ''}, 'discriminator_scheduler': {'warm_up_steps': ['0'], 'cycle_lengths': ['10000000000'], 'f_start': ['1e-06'], 'f_max': ['1.0'], 'f_min': ['1.0'], '_target_': ""}, 'student_update_freq': '5', 'gan_loss_weight_gen': '0', 'gan_use_same_t_noise': 'False', 'fake_score_pred_type': 'x0', 'gan_r1_reg_weight': '0.0', 'gan_r1_reg_alpha': '0.1', 'enable_gradient_in_rollout': 'True', 'start_gradient_frame': '0', 'same_step_across_blocks': 'True', 'last_step_only': 'False', 'context_noise': '0.0', 'fake_score': None, 'reward': {'enabled': 'True', 'checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/syncnet_v2.model', 'input_fps': '25.0', 'audio_sample_rate': '16000', 'vshift': '15', 'decoder_kind': 'taew', 'taew_checkpoint_path': '/home/work/.local/eval_metrics/checkpoints/auxiliary/taew2_1.pth'}, 'reward_beta': '2', 'center_reward': 'False', 'clamp_reward': None, 'reward_weighting_mode': 'per_sample', 'save_reward_debug_video': 'False', 'reward_debug_dir': 'logs/redmd_debug'}, 'model_class': {'config': None, '_target_': ""}, '_wandb': {}} +2026-04-20 00:28:42,140 INFO MainThread:2230976 [wandb_init.py:init():892] starting backend +2026-04-20 00:28:42,383 INFO MainThread:2230976 [wandb_init.py:init():895] sending inform_init request +2026-04-20 00:28:42,396 INFO MainThread:2230976 [wandb_init.py:init():903] backend started and connected +2026-04-20 00:28:42,399 INFO MainThread:2230976 [wandb_init.py:init():973] updated telemetry +2026-04-20 00:28:42,421 INFO MainThread:2230976 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-04-20 00:28:43,491 INFO MainThread:2230976 [wandb_init.py:init():1042] starting run threads in backend +2026-04-20 00:28:43,785 INFO MainThread:2230976 [wandb_run.py:_console_start():2524] atexit reg +2026-04-20 00:28:43,785 INFO MainThread:2230976 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-04-20 00:28:43,785 INFO MainThread:2230976 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-04-20 00:28:43,785 INFO MainThread:2230976 [wandb_run.py:_redirect():2465] Redirects installed. +2026-04-20 00:28:43,790 INFO MainThread:2230976 [wandb_init.py:init():1082] run started, returning control to user process diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/run-fxpzd9v7.wandb b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/run-fxpzd9v7.wandb new file mode 100644 index 0000000000000000000000000000000000000000..1a6217587567bb25fc86dcc2cc77744663e73e11 --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb/run-20260420_002842-fxpzd9v7/run-fxpzd9v7.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7fac77e50fbe88f6c60fec8347d523b9ea86539caf5be1f08d7bfd42d7aa7ce +size 11567104 diff --git a/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb_id.txt b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb_id.txt new file mode 100644 index 0000000000000000000000000000000000000000..8e6801e3a55806eb58e7b40e30682e465e40f69a --- /dev/null +++ b/FASTGEN_SF_OUTPUT_BETA2_AUDIOFIX_TAEW_4STEP/OmniAvatar-FastGen/omniavatar_sf_audiofix/sf_sink1_window7_redmd_audiofix_beta2_taew_4step/wandb_id.txt @@ -0,0 +1 @@ +fxpzd9v7